gtmharness 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gtmharness-0.5.0/LICENSE +21 -0
- gtmharness-0.5.0/PKG-INFO +147 -0
- gtmharness-0.5.0/README.md +132 -0
- gtmharness-0.5.0/gtmharness/adapters/__init__.py +27 -0
- gtmharness-0.5.0/gtmharness/adapters/enrich.py +198 -0
- gtmharness-0.5.0/gtmharness/adapters/hubspot.py +37 -0
- gtmharness-0.5.0/gtmharness/adapters/salesforce.py +35 -0
- gtmharness-0.5.0/gtmharness/adapters/webhook.py +27 -0
- gtmharness-0.5.0/gtmharness/audit.py +79 -0
- gtmharness-0.5.0/gtmharness/cli.py +318 -0
- gtmharness-0.5.0/gtmharness/doctor.py +78 -0
- gtmharness-0.5.0/gtmharness/dryrun.py +73 -0
- gtmharness-0.5.0/gtmharness/ledger.py +50 -0
- gtmharness-0.5.0/gtmharness/rollback.py +54 -0
- gtmharness-0.5.0/gtmharness/schema.py +57 -0
- gtmharness-0.5.0/gtmharness/skills.py +50 -0
- gtmharness-0.5.0/gtmharness/sources.py +21 -0
- gtmharness-0.5.0/gtmharness/spec.py +106 -0
- gtmharness-0.5.0/gtmharness/specdiff.py +78 -0
- gtmharness-0.5.0/gtmharness.egg-info/PKG-INFO +147 -0
- gtmharness-0.5.0/gtmharness.egg-info/SOURCES.txt +41 -0
- gtmharness-0.5.0/gtmharness.egg-info/dependency_links.txt +1 -0
- gtmharness-0.5.0/gtmharness.egg-info/entry_points.txt +2 -0
- gtmharness-0.5.0/gtmharness.egg-info/requires.txt +7 -0
- gtmharness-0.5.0/gtmharness.egg-info/top_level.txt +1 -0
- gtmharness-0.5.0/pyproject.toml +22 -0
- gtmharness-0.5.0/setup.cfg +4 -0
- gtmharness-0.5.0/tests/test_adapters.py +37 -0
- gtmharness-0.5.0/tests/test_audit.py +21 -0
- gtmharness-0.5.0/tests/test_audit_chain.py +51 -0
- gtmharness-0.5.0/tests/test_cli.py +77 -0
- gtmharness-0.5.0/tests/test_cli_m4.py +154 -0
- gtmharness-0.5.0/tests/test_doctor.py +46 -0
- gtmharness-0.5.0/tests/test_dryrun.py +45 -0
- gtmharness-0.5.0/tests/test_enrich.py +56 -0
- gtmharness-0.5.0/tests/test_ledger.py +27 -0
- gtmharness-0.5.0/tests/test_m4_fixes.py +139 -0
- gtmharness-0.5.0/tests/test_rollback.py +32 -0
- gtmharness-0.5.0/tests/test_schema.py +38 -0
- gtmharness-0.5.0/tests/test_skills.py +28 -0
- gtmharness-0.5.0/tests/test_spec.py +75 -0
- gtmharness-0.5.0/tests/test_specdiff.py +81 -0
- gtmharness-0.5.0/tests/test_transforms.py +146 -0
gtmharness-0.5.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gtmharness
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: CI/CD discipline for GTM pipelines: declared-as-code workflows with contract tests, dry-run diffs, and audit logs.
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: PyYAML>=6.0
|
|
10
|
+
Provides-Extra: hubspot
|
|
11
|
+
Requires-Dist: hubspot-api-python>=11.0; extra == "hubspot"
|
|
12
|
+
Provides-Extra: salesforce
|
|
13
|
+
Requires-Dist: simple-salesforce>=1.12; extra == "salesforce"
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# gtmharness
|
|
17
|
+
|
|
18
|
+
[](https://github.com/Yayihie/gtm-harness/actions/workflows/test.yml)
|
|
19
|
+
[](https://pypi.org/project/gtmharness/)
|
|
20
|
+
[](LICENSE)
|
|
21
|
+
|
|
22
|
+
**CI/CD discipline for GTM pipelines.** GTM workflows get built in no-code tools,
|
|
23
|
+
handed over as "a Loom video and a Clay table nobody else understands," and break
|
|
24
|
+
silently six months later when a field gets renamed upstream. gtmharness applies
|
|
25
|
+
the engineering practices that already fixed software to GTM:
|
|
26
|
+
|
|
27
|
+
- **Pipelines as code** — declared in YAML, validated at load time
|
|
28
|
+
- **Contract tests** — sinks declare field contracts (required, type, picklist enum); violations block the run
|
|
29
|
+
- **Schema drift detection** — snapshot source schemas; a renamed/removed/re-typed field fails loudly instead of quietly breaking routing
|
|
30
|
+
- **Dry-run plans** — `plan` shows the exact write set (WRITE / BLOCK per row) before anything touches the CRM
|
|
31
|
+
- **Tamper-evident audit log** — every write is logged with a rolling SHA-256 hash chain; `audit --verify` proves the log is intact
|
|
32
|
+
- **Enrichment waterfall** — BYO provider keys, cost-ordered, cached (never pay for the same lead twice), budget-capped per run
|
|
33
|
+
- **Rollback** — every run snapshots its write set; `revert` restores
|
|
34
|
+
|
|
35
|
+
Built from real Reddit pain in r/gtmengineering, r/revops, and r/marketing:
|
|
36
|
+
|
|
37
|
+
> "I've seen too many 'GTM OS' builds handed over as a Loom video and a Clay table nobody else understands. Six months later a field gets renamed upstream and the routing quietly breaks."
|
|
38
|
+
|
|
39
|
+
> "Most attribution tools fail not because of the tool but because the underlying CRM data isn't structured to feed them."
|
|
40
|
+
|
|
41
|
+
> "If it writes under the rep's name, you can't filter its changes in reports, you can't audit what it touched."
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install -e .
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Quickstart
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
# 1. Declare a pipeline (see examples/pipeline.yaml)
|
|
53
|
+
# 2. Check spec + detect schema drift
|
|
54
|
+
gtmharness check examples/pipeline.yaml
|
|
55
|
+
|
|
56
|
+
# 3. Dry-run: see exactly what would be written
|
|
57
|
+
gtmharness plan examples/pipeline.yaml
|
|
58
|
+
|
|
59
|
+
# 4. Run — writes only if the plan passes; everything audited
|
|
60
|
+
gtmharness run examples/pipeline.yaml
|
|
61
|
+
|
|
62
|
+
# 5. Read the audit log
|
|
63
|
+
gtmharness audit examples/pipeline.yaml
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Example `plan` output:
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
PLAN pipeline=inbound_lead_routing rows_in=3 writes=3 errors=0
|
|
70
|
+
[WRITE ] to_crm <- jane@acme.com {'email': 'jane@acme.com', 'lead_source': 'paid', ...}
|
|
71
|
+
[BLOCK ] to_crm <- bad@x.com {'email': 'bad@x.com', 'lead_source': 'free'}
|
|
72
|
+
! field 'lead_source': value 'free' not in enum ['paid', 'organic', 'event', 'referral']
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Philosophy
|
|
76
|
+
|
|
77
|
+
- **Local-first, zero-credit.** The core never talks to a paid API. Bring your own
|
|
78
|
+
provider keys via source adapters — no Clay-style credit metering.
|
|
79
|
+
- **Fail loudly, write nothing.** A blocked run writes zero rows. No partial state.
|
|
80
|
+
- **The contract is the documentation.** No Loom video required.
|
|
81
|
+
|
|
82
|
+
## Enrichment waterfall — BYO keys, cost-ordered, budget-capped
|
|
83
|
+
|
|
84
|
+
```yaml
|
|
85
|
+
- name: enrich
|
|
86
|
+
kind: transform
|
|
87
|
+
using: enrich
|
|
88
|
+
config:
|
|
89
|
+
providers: [prospeo, leadmagic] # cheapest first; falls through on miss
|
|
90
|
+
provider_config: {PROSPEO_API_KEY: "…", LEADMAGIC_API_KEY: "…"}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
- Cost-ordered: try the cheapest provider first, fall through only on a miss
|
|
94
|
+
- Per-lead cache: the same lead is never purchased twice (cached per state dir)
|
|
95
|
+
- Budget cap: `gtmharness run pipe.yaml --budget 5.00` aborts the run the moment
|
|
96
|
+
cumulative spend crosses the cap, with a ledger summary
|
|
97
|
+
- Every provider call is logged to the ledger with its real price
|
|
98
|
+
|
|
99
|
+
## Webhook sinks
|
|
100
|
+
|
|
101
|
+
```yaml
|
|
102
|
+
- name: fan_out
|
|
103
|
+
kind: sink
|
|
104
|
+
using: webhook
|
|
105
|
+
config: {url: "https://hooks.example.com/leads"}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Skills — reusable pipeline templates
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
gtmharness skills # list built-in + user skills
|
|
112
|
+
gtmharness skill inbound_lead_routing --set source_path=leads.json crm=crm
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Skills are plain YAML with `{{param}}` slots — drop your own into
|
|
116
|
+
`~/.gtmharness/skills/` and share them as files. No marketplace, no lock-in.
|
|
117
|
+
|
|
118
|
+
## Real CRM sinks (optional extras)
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
pip install "gtmharness[hubspot]" # official HubSpot SDK
|
|
122
|
+
pip install "gtmharness[salesforce]" # simple-salesforce
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Point a sink at `using: hubspot` or `using: salesforce` with your credentials in
|
|
126
|
+
the step config. The dry-run plan and audit log still gate every write — the
|
|
127
|
+
adapter is only the final "do it" step.
|
|
128
|
+
|
|
129
|
+
## Provider cost ledger
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
gtmharness ledger
|
|
133
|
+
# prospeo: 14 calls, 14 rows, $0.1400
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
Every BYO provider call is logged with its real cost to `ledger.jsonl` —
|
|
137
|
+
total transparency about what enrichment actually costs you.
|
|
138
|
+
|
|
139
|
+
## Status
|
|
140
|
+
|
|
141
|
+
v0.3.0 — spec validation, schema drift, dry-run plans, audit log, CSV/JSON sources,
|
|
142
|
+
skills registry, HubSpot + Salesforce + webhook sink adapters, enrichment waterfall
|
|
143
|
+
(BYO keys, cost-ordered, cached, budget-capped), provider cost ledger, GitHub Action.
|
|
144
|
+
|
|
145
|
+
Roadmap: see MILESTONES.md (M4: trust & ops — diff, doctor, rollback, tamper-evident audit log).
|
|
146
|
+
|
|
147
|
+
MIT licensed.
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# gtmharness
|
|
2
|
+
|
|
3
|
+
[](https://github.com/Yayihie/gtm-harness/actions/workflows/test.yml)
|
|
4
|
+
[](https://pypi.org/project/gtmharness/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
**CI/CD discipline for GTM pipelines.** GTM workflows get built in no-code tools,
|
|
8
|
+
handed over as "a Loom video and a Clay table nobody else understands," and break
|
|
9
|
+
silently six months later when a field gets renamed upstream. gtmharness applies
|
|
10
|
+
the engineering practices that already fixed software to GTM:
|
|
11
|
+
|
|
12
|
+
- **Pipelines as code** — declared in YAML, validated at load time
|
|
13
|
+
- **Contract tests** — sinks declare field contracts (required, type, picklist enum); violations block the run
|
|
14
|
+
- **Schema drift detection** — snapshot source schemas; a renamed/removed/re-typed field fails loudly instead of quietly breaking routing
|
|
15
|
+
- **Dry-run plans** — `plan` shows the exact write set (WRITE / BLOCK per row) before anything touches the CRM
|
|
16
|
+
- **Tamper-evident audit log** — every write is logged with a rolling SHA-256 hash chain; `audit --verify` proves the log is intact
|
|
17
|
+
- **Enrichment waterfall** — BYO provider keys, cost-ordered, cached (never pay for the same lead twice), budget-capped per run
|
|
18
|
+
- **Rollback** — every run snapshots its write set; `revert` restores
|
|
19
|
+
|
|
20
|
+
Built from real Reddit pain in r/gtmengineering, r/revops, and r/marketing:
|
|
21
|
+
|
|
22
|
+
> "I've seen too many 'GTM OS' builds handed over as a Loom video and a Clay table nobody else understands. Six months later a field gets renamed upstream and the routing quietly breaks."
|
|
23
|
+
|
|
24
|
+
> "Most attribution tools fail not because of the tool but because the underlying CRM data isn't structured to feed them."
|
|
25
|
+
|
|
26
|
+
> "If it writes under the rep's name, you can't filter its changes in reports, you can't audit what it touched."
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install -e .
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Quickstart
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
# 1. Declare a pipeline (see examples/pipeline.yaml)
|
|
38
|
+
# 2. Check spec + detect schema drift
|
|
39
|
+
gtmharness check examples/pipeline.yaml
|
|
40
|
+
|
|
41
|
+
# 3. Dry-run: see exactly what would be written
|
|
42
|
+
gtmharness plan examples/pipeline.yaml
|
|
43
|
+
|
|
44
|
+
# 4. Run — writes only if the plan passes; everything audited
|
|
45
|
+
gtmharness run examples/pipeline.yaml
|
|
46
|
+
|
|
47
|
+
# 5. Read the audit log
|
|
48
|
+
gtmharness audit examples/pipeline.yaml
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Example `plan` output:
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
PLAN pipeline=inbound_lead_routing rows_in=3 writes=3 errors=0
|
|
55
|
+
[WRITE ] to_crm <- jane@acme.com {'email': 'jane@acme.com', 'lead_source': 'paid', ...}
|
|
56
|
+
[BLOCK ] to_crm <- bad@x.com {'email': 'bad@x.com', 'lead_source': 'free'}
|
|
57
|
+
! field 'lead_source': value 'free' not in enum ['paid', 'organic', 'event', 'referral']
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Philosophy
|
|
61
|
+
|
|
62
|
+
- **Local-first, zero-credit.** The core never talks to a paid API. Bring your own
|
|
63
|
+
provider keys via source adapters — no Clay-style credit metering.
|
|
64
|
+
- **Fail loudly, write nothing.** A blocked run writes zero rows. No partial state.
|
|
65
|
+
- **The contract is the documentation.** No Loom video required.
|
|
66
|
+
|
|
67
|
+
## Enrichment waterfall — BYO keys, cost-ordered, budget-capped
|
|
68
|
+
|
|
69
|
+
```yaml
|
|
70
|
+
- name: enrich
|
|
71
|
+
kind: transform
|
|
72
|
+
using: enrich
|
|
73
|
+
config:
|
|
74
|
+
providers: [prospeo, leadmagic] # cheapest first; falls through on miss
|
|
75
|
+
provider_config: {PROSPEO_API_KEY: "…", LEADMAGIC_API_KEY: "…"}
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
- Cost-ordered: try the cheapest provider first, fall through only on a miss
|
|
79
|
+
- Per-lead cache: the same lead is never purchased twice (cached per state dir)
|
|
80
|
+
- Budget cap: `gtmharness run pipe.yaml --budget 5.00` aborts the run the moment
|
|
81
|
+
cumulative spend crosses the cap, with a ledger summary
|
|
82
|
+
- Every provider call is logged to the ledger with its real price
|
|
83
|
+
|
|
84
|
+
## Webhook sinks
|
|
85
|
+
|
|
86
|
+
```yaml
|
|
87
|
+
- name: fan_out
|
|
88
|
+
kind: sink
|
|
89
|
+
using: webhook
|
|
90
|
+
config: {url: "https://hooks.example.com/leads"}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Skills — reusable pipeline templates
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
gtmharness skills # list built-in + user skills
|
|
97
|
+
gtmharness skill inbound_lead_routing --set source_path=leads.json crm=crm
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Skills are plain YAML with `{{param}}` slots — drop your own into
|
|
101
|
+
`~/.gtmharness/skills/` and share them as files. No marketplace, no lock-in.
|
|
102
|
+
|
|
103
|
+
## Real CRM sinks (optional extras)
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
pip install "gtmharness[hubspot]" # official HubSpot SDK
|
|
107
|
+
pip install "gtmharness[salesforce]" # simple-salesforce
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Point a sink at `using: hubspot` or `using: salesforce` with your credentials in
|
|
111
|
+
the step config. The dry-run plan and audit log still gate every write — the
|
|
112
|
+
adapter is only the final "do it" step.
|
|
113
|
+
|
|
114
|
+
## Provider cost ledger
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
gtmharness ledger
|
|
118
|
+
# prospeo: 14 calls, 14 rows, $0.1400
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Every BYO provider call is logged with its real cost to `ledger.jsonl` —
|
|
122
|
+
total transparency about what enrichment actually costs you.
|
|
123
|
+
|
|
124
|
+
## Status
|
|
125
|
+
|
|
126
|
+
v0.3.0 — spec validation, schema drift, dry-run plans, audit log, CSV/JSON sources,
|
|
127
|
+
skills registry, HubSpot + Salesforce + webhook sink adapters, enrichment waterfall
|
|
128
|
+
(BYO keys, cost-ordered, cached, budget-capped), provider cost ledger, GitHub Action.
|
|
129
|
+
|
|
130
|
+
Roadmap: see MILESTONES.md (M4: trust & ops — diff, doctor, rollback, tamper-evident audit log).
|
|
131
|
+
|
|
132
|
+
MIT licensed.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Adapters package: optional provider integrations + a plugin registry.
|
|
2
|
+
|
|
3
|
+
Core gtm-harness has zero hard deps. CRM/enrichment adapters import their SDK
|
|
4
|
+
lazily and raise a helpful error if the SDK is not installed:
|
|
5
|
+
|
|
6
|
+
pip install gtmharness[hubspot]
|
|
7
|
+
pip install gtmharness[salesforce]
|
|
8
|
+
|
|
9
|
+
Plugin registry — custom sinks can be registered programmatically:
|
|
10
|
+
|
|
11
|
+
from gtmharness.adapters import register
|
|
12
|
+
|
|
13
|
+
register("my_sink", lambda config, rows: [{"key": r["email"], "id": None,
|
|
14
|
+
"status": "posted"} for r in rows])
|
|
15
|
+
|
|
16
|
+
Then use `using: my_sink` in a pipeline spec (and add it to spec.VALID_SINKS
|
|
17
|
+
when registering programmatically, e.g. in tests or a sitecustomize hook).
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from typing import Callable
|
|
21
|
+
|
|
22
|
+
REGISTRY: dict[str, Callable] = {}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def register(name: str, fn: Callable) -> None:
|
|
26
|
+
"""Register a sink adapter under `name`; fn(config, rows) -> list[dict]."""
|
|
27
|
+
REGISTRY[name] = fn
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Enrichment waterfall adapter — BYO provider keys, cost-ordered, MIT clean-room.
|
|
2
|
+
|
|
3
|
+
Following the pattern validated by openenrich (AGPL — no code reused):
|
|
4
|
+
|
|
5
|
+
1. Order providers by cost (cheapest first).
|
|
6
|
+
2. Call provider 1; if it returns a verified email, done.
|
|
7
|
+
3. Otherwise fall through to the next provider, skipping any lead a provider
|
|
8
|
+
already answered (never buy the same person twice).
|
|
9
|
+
4. Every call is logged to the cost ledger with its real price.
|
|
10
|
+
|
|
11
|
+
Built-in provider registry (BYO keys; each adapter module defines `lookup(lead, config)`):
|
|
12
|
+
|
|
13
|
+
name cost/hit key config
|
|
14
|
+
prospeo $0.01 PROSPEO_API_KEY
|
|
15
|
+
leadmagic $0.02 LEADMAGIC_API_KEY
|
|
16
|
+
null $0.00 free dry-run provider that always misses
|
|
17
|
+
|
|
18
|
+
The registry is a dict of provider name -> {"cost": float, "lookup": fn}.
|
|
19
|
+
Users add providers by registering in PROVIDERS (or a future plugin entry point).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
import time
|
|
26
|
+
import urllib.request
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
from .. import ledger
|
|
31
|
+
|
|
32
|
+
RETRY_ATTEMPTS = 3
|
|
33
|
+
RETRY_BACKOFF_S = 1.0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _http_json(url: str, api_key: str, payload: dict) -> dict:
|
|
37
|
+
req = urllib.request.Request(
|
|
38
|
+
url,
|
|
39
|
+
data=json.dumps(payload).encode(),
|
|
40
|
+
headers={"Content-Type": "application/json", "Authorization": api_key},
|
|
41
|
+
method="POST",
|
|
42
|
+
)
|
|
43
|
+
last_err = None
|
|
44
|
+
for attempt in range(RETRY_ATTEMPTS):
|
|
45
|
+
try:
|
|
46
|
+
with urllib.request.urlopen(req, timeout=30) as r:
|
|
47
|
+
return json.loads(r.read().decode())
|
|
48
|
+
except Exception as e: # transient network/API errors: retry with backoff
|
|
49
|
+
last_err = e
|
|
50
|
+
if attempt < RETRY_ATTEMPTS - 1:
|
|
51
|
+
time.sleep(RETRY_BACKOFF_S * (2 ** attempt))
|
|
52
|
+
raise last_err
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class WaterfallResult:
|
|
57
|
+
email: str | None = None
|
|
58
|
+
provider: str | None = None
|
|
59
|
+
cost_usd: float = 0.0
|
|
60
|
+
attempts: list[dict] = field(default_factory=list)
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def ok(self) -> bool:
|
|
64
|
+
return self.email is not None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ---------------------------------------------------------------- providers
|
|
68
|
+
|
|
69
|
+
def _lookup_prospeo(lead: dict, config: dict) -> dict:
|
|
70
|
+
"""{email, cost} — real call on your key."""
|
|
71
|
+
key = config.get("PROSPEO_API_KEY")
|
|
72
|
+
if not key:
|
|
73
|
+
raise RuntimeError("prospeo: set PROSPEO_API_KEY in provider config")
|
|
74
|
+
data = _http_json(
|
|
75
|
+
"https://api.prospeo.io/api/email-finder",
|
|
76
|
+
key,
|
|
77
|
+
{"name": lead.get("name", ""), "company": lead.get("company", "")},
|
|
78
|
+
)
|
|
79
|
+
return {"email": data.get("email"), "cost": 0.01}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _lookup_leadmagic(lead: dict, config: dict) -> dict:
|
|
83
|
+
key = config.get("LEADMAGIC_API_KEY")
|
|
84
|
+
if not key:
|
|
85
|
+
raise RuntimeError("leadmagic: set LEADMAGIC_API_KEY in provider config")
|
|
86
|
+
data = _http_json(
|
|
87
|
+
"https://api.leadmagic.io/email-finder",
|
|
88
|
+
key,
|
|
89
|
+
{"first_name": lead.get("first_name", ""), "last_name": lead.get("last_name", ""),
|
|
90
|
+
"domain": lead.get("domain", "")},
|
|
91
|
+
)
|
|
92
|
+
return {"email": data.get("email"), "cost": 0.02}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _lookup_null(lead: dict, config: dict) -> dict:
|
|
96
|
+
"""Free dry-run provider: always misses, never costs anything."""
|
|
97
|
+
return {"email": None, "cost": 0.0}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
PROVIDERS = {
|
|
101
|
+
"prospeo": {"cost": 0.01, "lookup": _lookup_prospeo},
|
|
102
|
+
"leadmagic": {"cost": 0.02, "lookup": _lookup_leadmagic},
|
|
103
|
+
"null": {"cost": 0.00, "lookup": _lookup_null},
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# ---------------------------------------------------------------- waterfall
|
|
108
|
+
|
|
109
|
+
def _lead_cache_key(lead: dict) -> str:
|
|
110
|
+
"""Stable key: name+company or email; identical leads never re-purchased."""
|
|
111
|
+
if lead.get("email"):
|
|
112
|
+
return f"email:{lead['email']}"
|
|
113
|
+
return f"person:{lead.get('name', '')}|{lead.get('company', '')}"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _cache_path(state_dir, pipeline) -> "Path":
|
|
117
|
+
d = Path(state_dir) / "enrich_cache"
|
|
118
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
119
|
+
return d / f"{pipeline or 'default'}.cache.json"
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def run_waterfall(lead: dict, provider_order: list[str], config: dict,
|
|
123
|
+
state_dir: str | None = None, pipeline: str | None = None) -> WaterfallResult:
|
|
124
|
+
"""Run one lead through the cost-ordered waterfall. Logs every call.
|
|
125
|
+
|
|
126
|
+
Cached leads (same email or name+company, seen in this state dir) return the
|
|
127
|
+
stored answer for free — never buy the same person twice.
|
|
128
|
+
"""
|
|
129
|
+
result = WaterfallResult()
|
|
130
|
+
key = _lead_cache_key(lead)
|
|
131
|
+
if state_dir is not None:
|
|
132
|
+
cf = _cache_path(state_dir, pipeline)
|
|
133
|
+
cache = json.loads(cf.read_text()) if cf.exists() else {}
|
|
134
|
+
if key in cache:
|
|
135
|
+
hit = cache[key]
|
|
136
|
+
result.email = hit.get("email")
|
|
137
|
+
result.provider = hit.get("provider")
|
|
138
|
+
result.attempts.append({"provider": hit.get("provider"), "status": "cached"})
|
|
139
|
+
return result
|
|
140
|
+
for name in provider_order:
|
|
141
|
+
if name not in PROVIDERS:
|
|
142
|
+
result.attempts.append({"provider": name, "status": "unknown provider"})
|
|
143
|
+
continue
|
|
144
|
+
spec = PROVIDERS[name]
|
|
145
|
+
try:
|
|
146
|
+
out = spec["lookup"](lead, config)
|
|
147
|
+
except Exception as e:
|
|
148
|
+
result.attempts.append({"provider": name, "status": f"error: {e}"})
|
|
149
|
+
continue
|
|
150
|
+
if state_dir is not None:
|
|
151
|
+
ledger.log_call(state_dir, name, "email_find", out.get("cost", spec["cost"]),
|
|
152
|
+
rows=1, pipeline=pipeline)
|
|
153
|
+
result.cost_usd += out.get("cost", spec["cost"])
|
|
154
|
+
if out.get("email"):
|
|
155
|
+
result.email = out["email"]
|
|
156
|
+
result.provider = name
|
|
157
|
+
result.attempts.append({"provider": name, "status": "hit"})
|
|
158
|
+
break
|
|
159
|
+
result.attempts.append({"provider": name, "status": "miss"})
|
|
160
|
+
if state_dir is not None:
|
|
161
|
+
cf = _cache_path(state_dir, pipeline)
|
|
162
|
+
cache = json.loads(cf.read_text()) if cf.exists() else {}
|
|
163
|
+
cache[key] = {"email": result.email, "provider": result.provider}
|
|
164
|
+
cf.write_text(json.dumps(cache, indent=2))
|
|
165
|
+
return result
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class BudgetExceeded(RuntimeError):
|
|
169
|
+
"""Raised when cumulative waterfall spend crosses the run budget."""
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def enrich_rows(rows: list[dict], provider_order: list[str], config: dict,
|
|
173
|
+
state_dir: str | None = None, pipeline: str | None = None,
|
|
174
|
+
budget_usd: float | None = None) -> list[dict]:
|
|
175
|
+
"""Enrich every row; adds email/lead_source on hit, marks misses.
|
|
176
|
+
|
|
177
|
+
budget_usd: hard cap on cumulative spend for this run. Crossing it raises
|
|
178
|
+
BudgetExceeded (callers can catch and print a ledger summary).
|
|
179
|
+
"""
|
|
180
|
+
out = []
|
|
181
|
+
spent = 0.0
|
|
182
|
+
for row in rows:
|
|
183
|
+
r = run_waterfall(row, provider_order, config, state_dir=state_dir, pipeline=pipeline)
|
|
184
|
+
spent += r.cost_usd
|
|
185
|
+
if budget_usd is not None and spent > budget_usd:
|
|
186
|
+
raise BudgetExceeded(
|
|
187
|
+
f"budget exceeded: spent ${spent:.4f} of ${budget_usd:.4f} cap "
|
|
188
|
+
f"after {len(out)} rows enriched"
|
|
189
|
+
)
|
|
190
|
+
enriched = dict(row)
|
|
191
|
+
if r.ok:
|
|
192
|
+
enriched["email"] = r.email
|
|
193
|
+
enriched["enrich_provider"] = r.provider
|
|
194
|
+
else:
|
|
195
|
+
enriched["enrich_status"] = "miss"
|
|
196
|
+
enriched["enrich_cost_usd"] = round(r.cost_usd, 4)
|
|
197
|
+
out.append(enriched)
|
|
198
|
+
return out
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""HubSpot sink adapter — wraps the official HubSpot Python SDK (optional extra).
|
|
2
|
+
|
|
3
|
+
Writes contacts/leads through the SDK's CRM API. Every write still goes through
|
|
4
|
+
the harness dry-run plan + audit log; this adapter is only the final "do it" step.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _client(config: dict):
|
|
11
|
+
token = config.get("token")
|
|
12
|
+
if not token:
|
|
13
|
+
raise RuntimeError("HubSpot sink config needs a 'token' (private app access token)")
|
|
14
|
+
try:
|
|
15
|
+
from hubspot import HubSpot # official hubspot-api-python # pyright: ignore
|
|
16
|
+
except ImportError as e:
|
|
17
|
+
raise RuntimeError(
|
|
18
|
+
"HubSpot adapter requires the official SDK: pip install 'gtmharness[hubspot]'"
|
|
19
|
+
) from e
|
|
20
|
+
return HubSpot(access_token=token)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def write_rows(config: dict, rows: list[dict]) -> list[dict]:
|
|
24
|
+
"""Write rows as HubSpot contacts. Returns [{'key', 'id', 'status'}]."""
|
|
25
|
+
client = _client(config)
|
|
26
|
+
out = []
|
|
27
|
+
for row in rows:
|
|
28
|
+
props = {k: v for k, v in row.items() if k in config.get("property_map", row.keys())}
|
|
29
|
+
props.setdefault("email", row.get("email"))
|
|
30
|
+
try:
|
|
31
|
+
created = client.crm.contacts.basic_api.create(
|
|
32
|
+
simple_public_object_input_for_create={"properties": props}
|
|
33
|
+
)
|
|
34
|
+
out.append({"key": row.get("email"), "id": created.id, "status": "created"})
|
|
35
|
+
except Exception as e:
|
|
36
|
+
out.append({"key": row.get("email"), "id": None, "status": f"error: {e}"})
|
|
37
|
+
return out
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Salesforce sink adapter — wraps simple-salesforce (optional extra).
|
|
2
|
+
|
|
3
|
+
Writes leads through the Salesforce REST API. Harness dry-run + audit still
|
|
4
|
+
gate the run; this adapter is only the final "do it" step.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _client(config: dict):
|
|
11
|
+
creds = {k: config.get(k) for k in ("username", "password", "security_token") if config.get(k)}
|
|
12
|
+
if len(creds) < 3:
|
|
13
|
+
raise RuntimeError("Salesforce sink config needs username, password, security_token")
|
|
14
|
+
try:
|
|
15
|
+
from simple_salesforce import Salesforce # MIT-licensed simple-salesforce # pyright: ignore
|
|
16
|
+
except ImportError as e:
|
|
17
|
+
raise RuntimeError(
|
|
18
|
+
"Salesforce adapter requires simple-salesforce: pip install 'gtmharness[salesforce]'"
|
|
19
|
+
) from e
|
|
20
|
+
return Salesforce(**creds)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def write_rows(config: dict, rows: list[dict]) -> list[dict]:
|
|
24
|
+
"""Write rows as Salesforce Leads. Returns [{'key', 'id', 'status'}]."""
|
|
25
|
+
client = _client(config)
|
|
26
|
+
out = []
|
|
27
|
+
for row in rows:
|
|
28
|
+
props = {k: v for k, v in row.items() if k in config.get("property_map", row.keys())}
|
|
29
|
+
try:
|
|
30
|
+
result = client.Lead.create(props)
|
|
31
|
+
out.append({"key": row.get("email"), "id": result.get("id"),
|
|
32
|
+
"status": "created" if result.get("success") else "error"})
|
|
33
|
+
except Exception as e:
|
|
34
|
+
out.append({"key": row.get("email"), "id": None, "status": f"error: {e}"})
|
|
35
|
+
return out
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Webhook sink adapter — POST rows to an HTTP endpoint.
|
|
2
|
+
|
|
3
|
+
Still gated by the dry-run plan + audit log; this is only the final "do it" step.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import urllib.request
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def write_rows(config: dict, rows: list[dict]) -> list[dict]:
|
|
13
|
+
url = config.get("url")
|
|
14
|
+
if not url:
|
|
15
|
+
raise RuntimeError("webhook sink config needs a 'url'")
|
|
16
|
+
headers = {"Content-Type": "application/json"}
|
|
17
|
+
if config.get("secret"):
|
|
18
|
+
headers["Authorization"] = f"Bearer {config['secret']}"
|
|
19
|
+
out = []
|
|
20
|
+
for row in rows:
|
|
21
|
+
try:
|
|
22
|
+
req = urllib.request.Request(url, data=json.dumps(row).encode(), headers=headers)
|
|
23
|
+
with urllib.request.urlopen(req, timeout=30) as r:
|
|
24
|
+
out.append({"key": row.get("email"), "id": None, "status": f"posted {r.status}"})
|
|
25
|
+
except Exception as e:
|
|
26
|
+
out.append({"key": row.get("email"), "id": None, "status": f"error: {e}"})
|
|
27
|
+
return out
|