thoughtleaders-cli 0.9.22__py3-none-any.whl → 0.9.24__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: thoughtleaders-cli
3
- Version: 0.9.22
3
+ Version: 0.9.24
4
4
  Summary: ThoughtLeaders CLI — query sponsorship data, channels, brands, and intelligence
5
5
  Project-URL: Homepage, https://thoughtleaders.io
6
6
  Project-URL: Repository, https://github.com/ThoughtLeaders-io/thoughtleaders-cli
@@ -1,4 +1,4 @@
1
- tl_cli/__init__.py,sha256=5HAz7aILtBFKk8rzV-S8W0GWF6ZtuH2pLbcdspC5h14,113
1
+ tl_cli/__init__.py,sha256=Fr4WPfFPO4tfJasNz2cfjOo8stYGSDkrJhJ8azhBL5o,113
2
2
  tl_cli/__main__.py,sha256=-FCBeOBi5Zi0hhZI3FWRilisl2DHxKeszgkY0r9VLR4,444
3
3
  tl_cli/_completions.py,sha256=kOyEUqC26vbYvyXWi513WX8fF73qQLR5WWuRSe_wqyk,164
4
4
  tl_cli/_typer_utils.py,sha256=ZiZsCVmEznPvBw-dYbr3tu3zWZ0iN6kjoQmK3gMqD28,860
@@ -24,7 +24,7 @@ tl_cli/commands/balance.py,sha256=s5uH0ijahT2gfgOKzOFxYxHyRBq50mT8zpNCY6KEsBY,37
24
24
  tl_cli/commands/brands.py,sha256=uogOfAdEbpjqiGlv4L2T1UC-Iz10G6fkRhVsDBOjRIY,15340
25
25
  tl_cli/commands/bulk_import.py,sha256=d4y1k_lD52LPJcCqXxEmyHIqcIwomZgbjqs1_QxPjeQ,4536
26
26
  tl_cli/commands/changelog.py,sha256=D1PtDdHpawTlWqUHjKzVmv9yXLSU915UVmI3dZzEwyA,4241
27
- tl_cli/commands/channels.py,sha256=gs6jHaxi6WjB3D8jnJETFVvkTXpqzlWR-W4JkBeG3oY,20406
27
+ tl_cli/commands/channels.py,sha256=ib1kTYNr0MjWVEbWLUmONX3-kyj0JLtwLGpTCxHwjHU,20409
28
28
  tl_cli/commands/credits.py,sha256=2xCht2e420LmaFBKNdKoMz8GlTh31qSWSlJAnVzoZic,7308
29
29
  tl_cli/commands/db.py,sha256=RcuGYzmTs5KMk6RgYao3oV-j883dG7IonTQclZg7UZs,9148
30
30
  tl_cli/commands/deals.py,sha256=ZK9yneInsC6DXoCPS65oyLoVR0eRW1xdRlEN7oRp1pc,2174
@@ -47,7 +47,7 @@ tl_cli/commands/workflows.py,sha256=ctduYa--Md6zbs8kQwO6mSyR9o1KB1UpNOaANuupDfU,
47
47
  tl_cli/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
48
48
  tl_cli/output/formatter.py,sha256=itNyHcDfRbksJgJ58JLtwJIrlHjqQj9gmK6uc3d9bSM,25692
49
49
  tl_cli/_plugin/.claude-plugin/marketplace.json,sha256=l56PMmyjfGXNGlV30wRyOAe74B6gJNCVNCxgsBbSNxc,446
50
- tl_cli/_plugin/.claude-plugin/plugin.json,sha256=eGjcDKuqbkjQUPkGQdEBaPpnQIAmjbdF5eua4qrRdBI,467
50
+ tl_cli/_plugin/.claude-plugin/plugin.json,sha256=RE1yHOU6LJMeA2u0DwL08urULLGRQ0XGFxq6srEvJD4,467
51
51
  tl_cli/_plugin/agents/cast-sheet.md,sha256=0owR8XIjR4tbKuK4t3WruiJAjGQ8se2cnOGwzigs4Lk,1224
52
52
  tl_cli/_plugin/agents/gem-classifier.md,sha256=s9kkbhUc-JhJ_be9z6LNaojykQSV16zrOd-QVaKNwxc,1250
53
53
  tl_cli/_plugin/agents/keyword-context-classifier.md,sha256=8Pk6WqJKuF-ERkBh54Aa-9t60CvjdOIH0RurFlG-T7I,4325
@@ -65,11 +65,11 @@ tl_cli/_plugin/skills/connector-impact/SKILL.md,sha256=K4mozNpRSLjTSICRTX8LihnKI
65
65
  tl_cli/_plugin/skills/connector-impact/references/help.md,sha256=Wd8X35OwYAswj-xoFZPnCWctwFIx9EXNZ511FDB2I5c,5831
66
66
  tl_cli/_plugin/skills/connector-impact/references/impact-schema.md,sha256=dlAtA6OwkqgXojaItnaQBEIXVxLoVuavuWsPtQ4Rc_0,10155
67
67
  tl_cli/_plugin/skills/connector-impact/references/impact-to-tl-mapping.md,sha256=58IJ5r1I0aEmIETFm81vMAbd1NxVMol8-fqqvHG3qXY,22122
68
- tl_cli/_plugin/skills/tl/SKILL.md,sha256=cYEGvIrjrKtBGsduwL3uuWkOdjFB6kW5iEzy1AMgnhE,72751
69
- tl_cli/_plugin/skills/tl/references/business-glossary.md,sha256=lNTHfo9zm7l_KAQB4libEVfpF-qZj7DXBlzKUo_uqkk,20208
68
+ tl_cli/_plugin/skills/tl/SKILL.md,sha256=C9XAkgeQUB-Or8-CSbZPVwoh9JS184bK0gj1y7nNiJ8,72757
69
+ tl_cli/_plugin/skills/tl/references/business-glossary.md,sha256=DkM-3DGnI_Dvr8fsB9B00-5erXv9r94umFCMVBGOxEs,20214
70
70
  tl_cli/_plugin/skills/tl/references/elasticsearch-schema.md,sha256=zjuHmJRooTTZvu2GfoQxG8xmRcExcKa33K1jipLAShI,25222
71
71
  tl_cli/_plugin/skills/tl/references/firebolt-schema.md,sha256=yf_elP_qXQePtMJNMshoVUOrI_3CXezF0NOoiYTbMNU,11335
72
- tl_cli/_plugin/skills/tl/references/postgres-schema.md,sha256=ardgaLIYTPD2HxNxTJP5AzN8tYUq5Ocu2sdit5JsBq0,30405
72
+ tl_cli/_plugin/skills/tl/references/postgres-schema.md,sha256=q0f264d1KhE96IwmVp7u-GX2N3tlanUiSIIPlqemIfo,30614
73
73
  tl_cli/_plugin/skills/tl-brand-creator-connection/SKILL.md,sha256=6__d4Z1_Mdb8m9nMKxU2wAYxAfu_3o_ucqn-0nXzqJs,35615
74
74
  tl_cli/_plugin/skills/tl-brand-creator-connection/references/cast-sheet.md,sha256=Y1Yj0D36tTGldn6uILoDOYwdMyjAp1oT9P-nUNgSyLg,4393
75
75
  tl_cli/_plugin/skills/tl-brand-creator-connection/references/creator-brief-template.md,sha256=mtFNSCPfj2fGrKtMX2g-Q1UNHYs59H4fARz320EutJA,10901
@@ -122,16 +122,16 @@ tl_cli/_plugin/skills/tl-create-workflow/references/creating-in-app.md,sha256=mJ
122
122
  tl_cli/_plugin/skills/tl-create-workflow/references/methodology.md,sha256=oXth-akS6QD7GmSDQRMaenAViTC5tlRgXIq_5uXVQQg,3847
123
123
  tl_cli/_plugin/skills/tl-create-workflow/references/pitfalls.md,sha256=6XBbe7T3o9PQcEs5FGNZ3n6xqVz0ZoNoFttJ9nNDjBU,3289
124
124
  tl_cli/_plugin/skills/tl-create-workflow/references/workflow-model.md,sha256=39cNg6AaSjb5K54neBkkn_RVJa7g9UnfZ_pVSzUsjDI,4591
125
- tl_cli/_plugin/skills/tl-keyword-research/SKILL.md,sha256=wvLvlumbR5fDxL-PZAqRDu7azYl4MXe1A5vOBn8YghE,38264
125
+ tl_cli/_plugin/skills/tl-keyword-research/SKILL.md,sha256=9KtLmWzPr93_i1dqrPnFx5yLC58EVoFGpIZJRG1kf6Y,41630
126
126
  tl_cli/_plugin/skills/tl-keyword-research/references/elasticsearch-content-search.md,sha256=fsXbpI3DPtkOnPEquB11Efur665o963RDN4-WIJCRJs,24263
127
127
  tl_cli/_plugin/skills/tl-keyword-research/references/help.md,sha256=yTmoU-Z6rPDjprbDa09nePoON0jnRSh_FVc7lqu1XGs,6802
128
- tl_cli/_plugin/skills/tl-keyword-research/scripts/build_report.py,sha256=oHCYfhFc006QhAv17R5uahpOMk38N1_YH_Cn6QMhQg4,22136
129
- tl_cli/_plugin/skills/tl-keyword-research/scripts/expand_entities.py,sha256=yYRfGRTl762Kvpv1xH_3nDZfShaX8BHoTRtmHyCsLk8,10886
128
+ tl_cli/_plugin/skills/tl-keyword-research/scripts/build_report.py,sha256=T_3xQ7vo_wdyZD5ObHrH5Lo_T-TcmBDygvXpfK-4P5I,23149
129
+ tl_cli/_plugin/skills/tl-keyword-research/scripts/expand_entities.py,sha256=r60FsvQiOsrCHG9xM1sGbzcN4kQTjWHSoiprVy61Ps4,12940
130
130
  tl_cli/_plugin/skills/tl-keyword-research/scripts/fetch_context.py,sha256=dn4n-WPjz4lib42hrDxrTN_xwqy1GZDpURE-IE6OKkg,6877
131
- tl_cli/_plugin/skills/tl-keyword-research/scripts/probe.py,sha256=eWmN8406QgrAcK_gvxNn63TX_NuF7Qk-X_CYK_MdMdU,28270
131
+ tl_cli/_plugin/skills/tl-keyword-research/scripts/probe.py,sha256=WiDdo39Wd0CXau2kn4Z5l__8dr9Spr2TkN3OCRPdj8o,31755
132
132
  tl_cli/_plugin/skills/tl-keyword-research/scripts/search_channels.py,sha256=qoe70bfpSWujO12eFP-OnfrqwbI7FgWTP_9aWavw3-U,23189
133
133
  tl_cli/_plugin/skills/tl-keyword-research/scripts/search_videos.py,sha256=Q_fYr1BzrMr5nh9wC1ruBJuIQFzNBkkBnYNMmsSEnc4,14568
134
- tl_cli/_plugin/skills/tl-keyword-research/scripts/select_keywords.py,sha256=eiavZfs-i2zAarR4k1_RT5wgqCr21T42S0AmwrXQid8,9359
134
+ tl_cli/_plugin/skills/tl-keyword-research/scripts/select_keywords.py,sha256=yQagYIN_Ql7AS-A02SwrUwOGEMNTyyms8hQYCcDZmPo,12791
135
135
  tl_cli/_plugin/skills/tl-save-report/SKILL.md,sha256=Sf1zIbLaJnCq2-IctIJrQDWWsTXJOTkkNzg4poc-wdo,37550
136
136
  tl_cli/_plugin/skills/tl-save-report/references/columns_brands.md,sha256=H5G308GOzMWvWw6smkPGdb2CBuuxkrojCpJgWS9X4sA,3559
137
137
  tl_cli/_plugin/skills/tl-save-report/references/columns_channels.md,sha256=g57ET_VI611lB7fVrGwYFvTDqMkNB63OmmfrnbMZF1I,4684
@@ -140,7 +140,7 @@ tl_cli/_plugin/skills/tl-save-report/references/columns_sponsorships.md,sha256=I
140
140
  tl_cli/_plugin/skills/tl-save-report/references/intelligence_filterset_schema.json,sha256=M_gUX6qDLUrAEcocYjU6ta6UaQS1kUSZpogCZzdqCTk,18140
141
141
  tl_cli/_plugin/skills/tl-save-report/references/intelligence_widget_schema.json,sha256=cxdlzmkt5WoI20zGsq_4qENTc6Ral9_vI5YTx2yzf-g,11503
142
142
  tl_cli/_plugin/skills/tl-save-report/references/report_glossary.md,sha256=YkUxEkUVui9HVC45BYBoyKVuyc_sLmf2GkSAEksmzW0,12165
143
- tl_cli/_plugin/skills/tl-save-report/references/sortable_columns.json,sha256=4X878VptWmS1ETfKZSXJ3U0E0lOw4JtVXsZTVHTsPUc,7571
143
+ tl_cli/_plugin/skills/tl-save-report/references/sortable_columns.json,sha256=Q8SYRqoMKMvu0r6qMWlSkmGdI_r5_Q9PwWhidqLFPig,7574
144
144
  tl_cli/_plugin/skills/tl-save-report/references/sponsorship_filterset_schema.json,sha256=7bS5G19A6JxOFyx-QoueURBoOf23FxCylwLeqgh8aKk,11381
145
145
  tl_cli/_plugin/skills/tl-save-report/references/sponsorship_widget_schema.json,sha256=PXEhZzRTNl94fFHpsBuo4Gop-Mdveg6GCaIm-ZGhBns,9587
146
146
  tl_cli/_plugin/skills/tl-save-report/references/widgets.md,sha256=LrqRyvZHOXVF15FglkeYh6io7f3cYmZEpH00WhEJWKw,9881
@@ -148,8 +148,8 @@ tl_cli/_plugin/skills/tl-top-partnerships/SKILL.md,sha256=jOMMr40XRnAZv-oRLeyKJT
148
148
  tl_cli/_plugin/skills/tl-top-partnerships/scripts/top_partnerships.py,sha256=OqhoyvFe1zrtrStChcam7oPYgFnSbGHC3WAgLhLIC3w,13738
149
149
  tl_cli/_plugin/skills/tl-views-guarantee/SKILL.md,sha256=IH7q1WJDWri9TWJMiga1FMGJO_GKSbWwaDS6CVNZ9c0,9270
150
150
  tl_cli/_plugin/skills/tl-views-guarantee/scripts/vg.py,sha256=5FucKm0WXa16btyUKMyGpdRQ5euVhzKMSoTonuwlm5I,16444
151
- thoughtleaders_cli-0.9.22.dist-info/METADATA,sha256=jPQ4Ht3HFhQ_7CcEQqEQxveY95gKC9mbdQbIWDmuHEM,22294
152
- thoughtleaders_cli-0.9.22.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
153
- thoughtleaders_cli-0.9.22.dist-info/entry_points.txt,sha256=umZp-1BkGkHDG0bNZXpTXrjwW0HGf9IDFN40eAWuuvg,39
154
- thoughtleaders_cli-0.9.22.dist-info/licenses/LICENSE,sha256=RUfdfLsn6jygiyrnnVUHt6r4IPwr2rbDm9Kixgtu8fo,1071
155
- thoughtleaders_cli-0.9.22.dist-info/RECORD,,
151
+ thoughtleaders_cli-0.9.24.dist-info/METADATA,sha256=ZEkuSp3hTzzSb7YHv8TR3OeGYUpI6f7ynkpRrBbsH9E,22294
152
+ thoughtleaders_cli-0.9.24.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
153
+ thoughtleaders_cli-0.9.24.dist-info/entry_points.txt,sha256=umZp-1BkGkHDG0bNZXpTXrjwW0HGf9IDFN40eAWuuvg,39
154
+ thoughtleaders_cli-0.9.24.dist-info/licenses/LICENSE,sha256=RUfdfLsn6jygiyrnnVUHt6r4IPwr2rbDm9Kixgtu8fo,1071
155
+ thoughtleaders_cli-0.9.24.dist-info/RECORD,,
tl_cli/__init__.py CHANGED
@@ -1,3 +1,3 @@
1
1
  """ThoughtLeaders CLI — query sponsorship data, channels, brands, and intelligence."""
2
2
 
3
- __version__ = "0.9.22"
3
+ __version__ = "0.9.24"
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tl-cli",
3
- "version": "0.9.22",
3
+ "version": "0.9.24",
4
4
  "description": "ThoughtLeaders CLI — query sponsorship deals, channels, brands, uploads, and intelligence from the terminal",
5
5
  "author": {
6
6
  "name": "ThoughtLeaders",
@@ -95,7 +95,7 @@ Other key concepts:
95
95
  - Org scoping for sponsorships is profile-mediated: a sponsorship belongs to your org if **either** `advertiser_profile.organization` (brand side) **or** the adspot seller's organization — `ad_spot.publisher` → that user's profile → `organization` (publisher side) — matches yours. Representing the channel does not surface its deals.
96
96
  - **MSN** (Media Selling Network) — the ~12k YouTube channels that have opted in to receive sponsorship offers. A channels is in the MSN group if the `channel.media_selling_network_join_date` field is not null.
97
97
  - **MBN** (Media Buying Network) — the brand-side counterpart to MSN: brand profiles that have opted in to receive proposed sponsorships. A profile is in the MBN group if the `profile.media_buying_network_join_date` field is not null.
98
- - **TPP** (ThoughtLeaders Partner Program, a.k.a. "TL channels") — the ~170 channels TL has the closest working relationship with. A channel is in the TPP group if `channel.is_tpp` is True. **Prefer TPP channels when booking**: they respond fastest, are the easiest to close, and don't need an outreach round-trip — treat them as immediately bookable. TPP is a strict subset of MSN, so the same booking rules (one active mention adspot, etc.) apply.
98
+ - **TPP** (ThoughtLeaders Partner Program, a.k.a. "TL channels") — the ~170 channels TL has the closest working relationship with. A channel is in the TPP group if `channel.is_tpp` is True. **Prefer TPP channels when booking**: they respond fastest, are the easiest to close, and don't need an outreach round-trip — treat them as immediately bookable. TPP is a strict subset of MSN, so the same booking rules (one preferred mention adspot, etc.) apply.
99
99
  - **`demographics_updated_at`** (on channels) — If non-null, the channel has demographics screenshots on file. If null, no demographics screenshots have been uploaded. Use this to check whether a channel has demographics data from screenshots.
100
100
  - **`subscribers`** (on channels) — the channel's subscriber count.
101
101
  - **`projected_views`** (on channels) — projected views per video on that channel. Forward-looking estimate. May be null when not yet computed. ⚠️ NOT actual views and NOT ad-industry "impressions" (ads served).
@@ -103,7 +103,7 @@ Other key concepts:
103
103
  - **`views_guarantee`** (on sponsorships) — projected/guaranteed impressions for the sponsorship. Numeric.
104
104
  - **Sponsorship detail fields** (returned by `tl sponsorships show <id> --json`) — the detail payload includes `integration` (raw int), `publish_count`, `common_name`, `outreach_email`, nested `publisher` (`first_name`, `last_name`, `email`), nested `brand_contact` (`first_name`, `last_name`, `email`), and `brand.organization_name`. Use these when generating IOs, contracts, or outreach.
105
105
  - **CPM** has two distinct meanings depending on level — pick the one the user actually wants:
106
- - **Channel CPM** = `(adspot.price / channel.projected_views) * 1000` — projected price per thousand projected views. Used for pricing decisions **before** a sponsorship is sold. Available for channels with active adspots via `tl channels show <channel_id>`.
106
+ - **Channel CPM** = `(adspot.price / channel.projected_views) * 1000` — projected price per thousand projected views. Used for pricing decisions **before** a sponsorship is sold. Available for channels with preferred adspots via `tl channels show <channel_id>`.
107
107
  - **Sponsorship CPM** = calculated in either of two ways: if `views` is present, then CPM is `(sponsorship.price / sponsorship.views) × 1000`, meaning realized cost per thousand actual views, computed post-publication. If `views` is null, Compute from the sponsorship's `price` and the channel's `projected_views` fields.
108
108
  - Where possible, calculate the correct CPM in a SQL expression.
109
109
  - **Sponsorship dates** — each sponsorship has four distinct dates, useful for different queries:
@@ -56,7 +56,7 @@ Maps business terms to database concepts.
56
56
  - A channel can have **multiple ad spots** because different people sell the same channel (talent manager, direct, multiple agencies)
57
57
  - Ad spots are the **catalogue** — adlinks are **instances** of catalogue items
58
58
  - Price/cost on adspot = list/catalog values; price/cost on adlink = actual deal values
59
- - **Only one active adspot with integration=mention per channel at any time** (MSN rule)
59
+ - **Only one preferred adspot with integration=mention per channel at any time** (MSN rule)
60
60
 
61
61
  ## Ownership & Accountability
62
62
 
@@ -79,7 +79,7 @@ Maps business terms to database concepts.
79
79
  - Key data: **who is the contact** to buy the ad from
80
80
  - `thoughtleaders_channel.media_selling_network_join_date` = when channel joined MSN
81
81
  - `thoughtleaders_channel.is_tpp` = TPP channel — TL's closest partners, a strict subset of MSN. **Prefer when booking**: fastest response, easiest to close. Not a gating flag; not a process barrier.
82
- - **Rule:** Only one active adspot with `integration=mention` per channel at any time
82
+ - **Rule:** Only one preferred adspot with `integration=mention` per channel at any time
83
83
  - MSN quality depends on having current, accurate contact info
84
84
 
85
85
  ## Channels & Audience
@@ -165,8 +165,9 @@ A channel can have multiple adspots (different sellers: talent manager, direct,
165
165
  | `channel_id` | int FK | → `thoughtleaders_channel.id` |
166
166
  | `price` | numeric | List/catalogue price |
167
167
  | `cost` | numeric | List/catalogue cost |
168
- | `integration` | int | 1=YouTube Mentions (live reads). Only one active mention-type adspot per channel. |
169
- | `is_active` | boolean | Active flag |
168
+ | `integration` | int | 1=YouTube Mentions (live reads). Only one preferred mention-type adspot per channel. |
169
+ | `is_preferred` | boolean | The channel's preferred spot, the one new proposals are booked on |
170
+ | `is_active` | boolean | False = deactivated, unusable, kept only for its AdLinks. Client sandboxes never show deactivated spots or their AdLinks |
170
171
  | `publisher_id` | int FK | → `auth_user.id` (NOT `thoughtleaders_profile.id` — see gotcha below) |
171
172
 
172
173
  ### Key columns for the `thoughtleaders_channel` table
@@ -23,6 +23,33 @@ description: |
23
23
  common is X". Also invoke for HELP asks about this skill — "help", "how
24
24
  does keyword research work", "what are my options", "describe this skill" —
25
25
  answered free from the built-in guide, no queries run.
26
+ output-schema:
27
+ type: object
28
+ required: [keyword_groups, keyword_operator, assumptions]
29
+ properties:
30
+ keyword_groups:
31
+ type: array
32
+ items:
33
+ type: object
34
+ required: [text]
35
+ properties:
36
+ text: {type: string}
37
+ content_fields:
38
+ type: array
39
+ items: {type: string}
40
+ exclude: {type: boolean}
41
+ keyword_operator: {type: string, enum: [OR, AND]}
42
+ assumptions:
43
+ type: array
44
+ items: {type: string}
45
+ dropped:
46
+ type: array
47
+ items:
48
+ type: object
49
+ required: [text, reason]
50
+ properties:
51
+ text: {type: string}
52
+ reason: {type: string}
26
53
  ---
27
54
 
28
55
  # tl-keyword-research — topic → validated filter set + the results you choose
@@ -69,6 +96,33 @@ verbatim via `search_channels.py --group` / `search_videos.py --group`.
69
96
 
70
97
  `<SKILL_DIR>` below is this skill's directory (the one holding `SKILL.md`).
71
98
 
99
+ ## On the ThoughtLeaders platform
100
+
101
+ The same skill also runs on the ThoughtLeaders platform, when a user starts
102
+ keyword research from their dashboard. You are on the platform when the tools
103
+ `kw_probe` and `kw_validate` are available. The method in this file applies
104
+ unchanged, with these differences:
105
+
106
+ - **No shell and no sub-agents.** Stages that run a script or an agent carry
107
+ an *On the platform* note naming the tool to use instead.
108
+ - **Nobody to ask.** The task gives you the path (quick or deep) and the
109
+ deliverable. Make every scope call yourself and record it in `assumptions`.
110
+ Run the deep path autonomously: no checkpoint after round 3.
111
+ - **Titles and summaries only.** Search the `title` and `summary` fields,
112
+ never `transcript`.
113
+ - **Stop after Stage 4.** The platform builds the trend data, channel tiers,
114
+ report link and previews from your final groups, so skip Stage 5 and the
115
+ scripts in Stage 6.
116
+ - **Decide, don't count.** Finish with `submit_result`, shaped by this
117
+ skill's `output-schema`: the final keyword groups, the operator, your
118
+ assumptions, and the groups you dropped with a reason. Never put counts or
119
+ channel figures in the result.
120
+ - **Narrate with `report_progress`.** Call it at the start of each stage with
121
+ the stage name exactly as the pipeline table writes it (`Set up`, `Expand`,
122
+ `Probe`, `Validate keywords`, `Refine`), instead of messaging the user.
123
+ - **References.** Read `references/elasticsearch-content-search.md` with
124
+ `read_skill_file`.
125
+
72
126
  ## When to invoke / skip
73
127
 
74
128
  Invoke when the user wants **videos or channels by content** (topics, concepts,
@@ -234,6 +288,10 @@ index's samples, and every web-sourced name earns its place through the same
234
288
  probe + validation as any other candidate (no benefit of the doubt; a
235
289
  hallucinated name probes to `count: 0` and drops).
236
290
 
291
+ *On the platform:* use `web_search` (at most two searches) in place of the
292
+ `keyword-entity-resolver` agent, then add each name and its spelling variants
293
+ to your candidates yourself, following Stage 1's rules.
294
+
237
295
  ### Stage 1 — Expand: go wide, go deep, go creative
238
296
 
239
297
  A shallow synonym list is the #1 failure mode. Coverage is won or lost here.
@@ -271,6 +329,11 @@ Rules:
271
329
 
272
330
  ### Stage 2 — Probe (`probe.py`)
273
331
 
332
+ *On the platform:* call `kw_probe` with the candidates and the same options
333
+ `probe.py` takes (`level`, `mode`, `fields`, `samples`, `content_type`,
334
+ `since`, `until`). It returns the counts; the samples stay with the platform
335
+ for `kw_validate`.
336
+
274
337
  One ES query per candidate → counts + samples for validation:
275
338
 
276
339
  ```bash
@@ -322,6 +385,11 @@ surviving variants before delivery.
322
385
  This fixes "the word is there but the topic isn't" — the step that separates
323
386
  this skill from a free YouTube search.
324
387
 
388
+ *On the platform:* call `kw_validate` on the probe with the verbatim intent. It
389
+ runs the relevance check and the strict-majority rule, and returns the kept
390
+ and dropped keywords. Scope calls are yours: decide them and record them in
391
+ `assumptions`.
392
+
325
393
  **Inline (≤ ~15 candidates):** read each keyword's `samples` against the
326
394
  verbatim intent. Drop off-intent keywords, `count: 0`, and redundant
327
395
  `subsumed_by` duplicates.
@@ -448,8 +516,14 @@ then fold the answers into the groups and the validators' TOPIC/NOT lines.
448
516
  run ≥3 rounds, stop when fitness stops improving (sane cap ~6 rounds), note
449
517
  that you ran autonomously.
450
518
 
519
+ *On the platform:* always run autonomously, measuring with `kw_probe` (mode
520
+ `sqs`) and re-validating with `kw_validate`.
521
+
451
522
  ### Stage 5 — Materialize the chosen deliverables
452
523
 
524
+ *On the platform:* skip this stage. The platform builds the deliverables from
525
+ your final groups.
526
+
453
527
  Everything here takes the final filter verbatim via `--group`, so what you
454
528
  deliver is exactly what the filter selects.
455
529
 
@@ -541,6 +615,11 @@ on-topic uploads. Tell the user when one channel dominates and offer
541
615
 
542
616
  ### Stage 6 — Deliver (`build_report.py`)
543
617
 
618
+ *On the platform:* don't run `build_report.py` and don't offer to save. Call
619
+ `submit_result` with the groups in the same shape (`text`, plus
620
+ `content_fields` / `exclude` where needed); the platform prunes them,
621
+ translates them into the report grammar and builds the link.
622
+
544
623
  Hand the validated groups to the builder — start from `select_keywords.py
545
624
  --apply`'s `groups`, adding per-group `content_fields` / `exclude` as needed:
546
625
 
@@ -39,7 +39,15 @@ Input (stdin JSON):
39
39
  }
40
40
 
41
41
  Output (stdout): a single JSON object — see OUTPUT CONTRACT at the bottom.
42
+
43
+ Library use: the file is import-safe (standard library only, nothing runs at import),
44
+ so another Python program can load it by path and call `build(spec, *, app_url=...)`.
45
+ `spec` is the parsed input object above; the return value is the object the command
46
+ prints. `build` leaves `spec` untouched and returns fresh structures. A bad spec raises
47
+ `ReportSpecError` (a `ValueError`). Library functions never print, read stdin or exit.
42
48
  """
49
+ from __future__ import annotations
50
+
43
51
  import argparse
44
52
  import json
45
53
  import re
@@ -116,6 +124,10 @@ DEFAULT_FIELDS = {
116
124
  }
117
125
 
118
126
 
127
+ class ReportSpecError(ValueError):
128
+ """The spec handed to `build` is unusable."""
129
+
130
+
119
131
  def tokens(text):
120
132
  return _TOKEN_RE.findall(text.lower())
121
133
 
@@ -389,33 +401,23 @@ def build_report_config(filter_set, report_type_int, title, description):
389
401
  }
390
402
 
391
403
 
392
- def main():
393
- ap = argparse.ArgumentParser(
394
- description="Build the keyword-research deliverable (filter set + report link) from validated groups."
395
- )
396
- ap.add_argument("--app-url", default=DEFAULT_APP_URL, help=f"App base URL (default {DEFAULT_APP_URL})")
397
- args = ap.parse_args()
398
-
399
- if sys.stdin.isatty():
400
- sys.exit("pipe a JSON spec on stdin (see module docstring)")
401
- try:
402
- spec = json.loads(sys.stdin.read())
403
- except json.JSONDecodeError as exc:
404
- sys.exit(f"invalid JSON on stdin: {exc}")
405
-
406
- operator = (spec.get("operator") or "OR").upper()
404
+ def build(spec, *, app_url=DEFAULT_APP_URL):
405
+ """Build the deliverable (filter set, report link, report config) from a spec; raises ReportSpecError."""
406
+ if not isinstance(spec, dict):
407
+ raise ReportSpecError("spec must be a JSON object")
408
+ operator = str(spec.get("operator") or "OR").upper()
407
409
  if operator not in ("AND", "OR"):
408
- sys.exit("operator must be AND or OR")
409
- report_type = (spec.get("report_type") or "channels").lower()
410
+ raise ReportSpecError("operator must be AND or OR")
411
+ report_type = str(spec.get("report_type") or "channels").lower()
410
412
  if report_type not in REPORT_TYPES:
411
- sys.exit(f"report_type must be one of {sorted(REPORT_TYPES)}")
413
+ raise ReportSpecError(f"report_type must be one of {sorted(REPORT_TYPES)}")
412
414
  slug, report_type_int = REPORT_TYPES[report_type]
413
415
  default_fields = spec.get("default_content_fields") or DEFAULT_FIELDS[report_type]
414
416
 
415
417
  groups = spec.get("groups") or []
416
- groups = [g for g in groups if isinstance(g, dict) and str(g.get("text", "")).strip()]
418
+ groups = [dict(g) for g in groups if isinstance(g, dict) and str(g.get("text", "")).strip()]
417
419
  if not groups:
418
- sys.exit("provide at least one group with non-empty 'text'")
420
+ raise ReportSpecError("provide at least one group with non-empty 'text'")
419
421
  for g in groups:
420
422
  g["text"] = str(g["text"]).strip()
421
423
 
@@ -426,7 +428,7 @@ def main():
426
428
  used_fields.update(g.get("content_fields") or [])
427
429
  unknown = sorted(f for f in used_fields if f not in VALID_CONTENT_FIELDS)
428
430
  if unknown:
429
- sys.exit(
431
+ raise ReportSpecError(
430
432
  f"unknown content_fields {unknown}; use recognized content field names "
431
433
  f"(e.g. title, summary, transcript, channel_description, "
432
434
  f"channel_topic_description) — not raw ES paths. Valid: {sorted(VALID_CONTENT_FIELDS)}"
@@ -443,18 +445,18 @@ def main():
443
445
  try:
444
446
  app_text = sqs_to_app_syntax(g["text"])
445
447
  except ValueError as exc:
446
- sys.exit(f"group {g['text']!r}: {exc}")
448
+ raise ReportSpecError(f"group {g['text']!r}: {exc}")
447
449
  if app_text != g["text"]:
448
450
  translated.append({"from": g["text"], "to": app_text})
449
451
  g["text"] = app_text
450
452
 
451
453
  filter_set = build_filter_set(kept, operator, default_fields)
452
- report_link = build_inline_link(kept, operator, slug, default_fields, args.app_url)
454
+ report_link = build_inline_link(kept, operator, slug, default_fields, app_url)
453
455
  title = spec.get("title") or f"{report_type.title()} — keyword search"
454
456
  description = spec.get("description") or ""
455
457
  report_config = build_report_config(filter_set, report_type_int, title, description)
456
458
 
457
- print(json.dumps({
459
+ return {
458
460
  "report_type": report_type,
459
461
  "operator": operator,
460
462
  "expression": render_expression(kept, operator),
@@ -463,7 +465,28 @@ def main():
463
465
  "report_config": report_config,
464
466
  "pruned": pruned,
465
467
  "translated": translated,
466
- }, ensure_ascii=False, indent=2))
468
+ }
469
+
470
+
471
+ def main():
472
+ ap = argparse.ArgumentParser(
473
+ description="Build the keyword-research deliverable (filter set + report link) from validated groups."
474
+ )
475
+ ap.add_argument("--app-url", default=DEFAULT_APP_URL, help=f"App base URL (default {DEFAULT_APP_URL})")
476
+ args = ap.parse_args()
477
+
478
+ if sys.stdin.isatty():
479
+ sys.exit("pipe a JSON spec on stdin (see module docstring)")
480
+ try:
481
+ spec = json.loads(sys.stdin.read())
482
+ except json.JSONDecodeError as exc:
483
+ sys.exit(f"invalid JSON on stdin: {exc}")
484
+
485
+ try:
486
+ out = build(spec, app_url=args.app_url)
487
+ except ReportSpecError as exc:
488
+ sys.exit(str(exc))
489
+ print(json.dumps(out, ensure_ascii=False, indent=2))
467
490
 
468
491
 
469
492
  # OUTPUT CONTRACT (stdout, single JSON object):
@@ -33,8 +33,19 @@ Usage:
33
33
  python3 expand_entities.py --existing "cannes lions" "advertising awards" < resolver.json > expanded.json
34
34
  # pipe the candidates straight into probe.py:
35
35
  python3 expand_entities.py --probe-batch < resolver.json | python3 probe.py --samples 5
36
+
37
+ Library use: the file is import-safe (standard library only, nothing runs at import),
38
+ so another Python program can load it by path and call `expand(resolver_reply, *,
39
+ existing=None)` (what the default mode prints) or `probe_batch(resolver_reply, *,
40
+ existing=None)` (what `--probe-batch` prints). `resolver_reply` is the parsed resolver
41
+ JSON; `existing` lists the candidates already brainstormed. Both return fresh structures
42
+ that share nothing with the input. Bad input raises `ExpandInputError` (a `ValueError`).
43
+ Library functions never print, read stdin or exit.
36
44
  """
45
+ from __future__ import annotations
46
+
37
47
  import argparse
48
+ import copy
38
49
  import json
39
50
  import re
40
51
  import sys
@@ -80,6 +91,10 @@ def solid(text):
80
91
  return _NONWORD_RE.sub("", text)
81
92
 
82
93
 
94
+ class ExpandInputError(ValueError):
95
+ """The resolver reply or options handed to the expander are unusable."""
96
+
97
+
83
98
  def _norm(text):
84
99
  """Case/space-insensitive key for dedupe."""
85
100
  return re.sub(r"\s+", " ", text.strip().lower())
@@ -156,8 +171,25 @@ def _collect_names(resolver, max_insider):
156
171
  return fams
157
172
 
158
173
 
159
- def expand(resolver, existing, max_insider):
160
- """Build probe candidates + family detail from a resolver JSON object."""
174
+ def check_resolver(resolver_reply):
175
+ """Raise ExpandInputError unless the resolver reply is a JSON object."""
176
+ if not isinstance(resolver_reply, dict):
177
+ raise ExpandInputError("resolver JSON must be an object")
178
+
179
+
180
+ def expand(resolver_reply, *, existing=None, max_insider=MAX_INSIDER_DEFAULT):
181
+ """Build probe candidates + family detail from a resolver JSON object (what the default mode prints).
182
+
183
+ `existing` lists candidates already brainstormed; matching forms are dropped.
184
+ `max_insider` caps the insider terms folded in. Raises ExpandInputError.
185
+ """
186
+ check_resolver(resolver_reply)
187
+ resolver = resolver_reply
188
+ existing = [] if existing is None else existing
189
+ if not isinstance(existing, (list, tuple)) or not all(isinstance(x, str) for x in existing):
190
+ raise ExpandInputError("existing must be a list of strings")
191
+ if max_insider < 0:
192
+ raise ExpandInputError("--max-insider must be >= 0")
161
193
  seen = {_norm(x) for x in existing} # forms already covered → skip
162
194
  deduped = []
163
195
 
@@ -211,7 +243,7 @@ def expand(resolver, existing, max_insider):
211
243
  )
212
244
  collisions = [c for c in (resolver.get("collisions", []) or []) if isinstance(c, dict)]
213
245
 
214
- return {
246
+ result = {
215
247
  "probe_candidates": probe_candidates,
216
248
  "families": families,
217
249
  "aliases": aliases,
@@ -228,6 +260,12 @@ def expand(resolver, existing, max_insider):
228
260
  "deduped": len(deduped),
229
261
  },
230
262
  }
263
+ return copy.deepcopy(result) # the caller's input is never shared with the output
264
+
265
+
266
+ def probe_batch(resolver_reply, *, existing=None, max_insider=MAX_INSIDER_DEFAULT):
267
+ """Just the probe_candidates array (what --probe-batch prints), ready for probe.py."""
268
+ return expand(resolver_reply, existing=existing, max_insider=max_insider)["probe_candidates"]
231
269
 
232
270
 
233
271
  def load_resolver():
@@ -240,8 +278,10 @@ def load_resolver():
240
278
  data = json.loads(raw)
241
279
  except json.JSONDecodeError as exc:
242
280
  sys.exit(f"invalid resolver JSON on stdin: {exc}")
243
- if not isinstance(data, dict):
244
- sys.exit("resolver JSON must be an object")
281
+ try:
282
+ check_resolver(data)
283
+ except ExpandInputError as exc:
284
+ sys.exit(str(exc))
245
285
  return data
246
286
 
247
287
 
@@ -259,7 +299,11 @@ def main():
259
299
  if args.max_insider < 0:
260
300
  sys.exit("--max-insider must be >= 0")
261
301
 
262
- out = expand(load_resolver(), args.existing, args.max_insider)
302
+ resolver = load_resolver()
303
+ try:
304
+ out = expand(resolver, existing=args.existing, max_insider=args.max_insider)
305
+ except ExpandInputError as exc:
306
+ sys.exit(str(exc))
263
307
  if args.probe_batch:
264
308
  print(json.dumps(out["probe_candidates"], ensure_ascii=False))
265
309
  else:
@@ -28,13 +28,24 @@ Usage:
28
28
  echo '[{"label":"shop+amazon","sqs":"\"tiktok shop\" +amazon"}]' | probe.py --mode sqs
29
29
 
30
30
  Output (stdout): a single JSON object — see OUTPUT CONTRACT at the bottom.
31
+
32
+ Library use: the file is import-safe (standard library only, nothing runs at import),
33
+ so another Python program can load it by path and call
34
+ `run_probe(candidates, *, run_es, fields, ...)`. It returns the same object the command
35
+ prints. `run_es(body) -> dict` is the caller's query function (an Elasticsearch body in,
36
+ the parsed response out); a candidate whose query raises is recorded under `failed`.
37
+ `fields` is required — the command's defaults live only in `main()`. Bad input raises
38
+ `ProbeInputError` (a `ValueError`). Library functions never print, read stdin or exit.
31
39
  """
40
+ from __future__ import annotations
41
+
32
42
  import argparse
33
43
  import datetime
34
44
  import json
35
45
  import re
36
46
  import subprocess
37
47
  import sys
48
+ from collections.abc import Callable
38
49
 
39
50
  # Article-level (doc_type:article) text fields vs channel-level (doc_type:channel).
40
51
  # `summary` holds the video's creator-written description (links, hashtags, promo)
@@ -140,12 +151,12 @@ def build_body(candidate, *, fields, level, samples, source_paths, since, until,
140
151
  match = {
141
152
  "simple_query_string": {
142
153
  "query": candidate["value"],
143
- "fields": fields,
154
+ "fields": list(fields),
144
155
  "default_operator": "and",
145
156
  }
146
157
  }
147
158
  else:
148
- match = {"multi_match": {"query": candidate["value"], "type": "phrase", "fields": fields}}
159
+ match = {"multi_match": {"query": candidate["value"], "type": "phrase", "fields": list(fields)}}
149
160
 
150
161
  # Always scope to YouTube uploads (format 4). At topic level also default to
151
162
  # longform videos (content_type is article-only; channel docs have none).
@@ -180,7 +191,7 @@ def build_body(candidate, *, fields, level, samples, source_paths, since, until,
180
191
  return {
181
192
  "size": samples,
182
193
  "track_total_hits": True,
183
- "_source": source_paths,
194
+ "_source": list(source_paths),
184
195
  "collapse": {"field": collapse_field},
185
196
  "aggs": aggs,
186
197
  "query": {"bool": {"filter": filters, "must": [match]}},
@@ -191,6 +202,10 @@ class ProbeError(Exception):
191
202
  """One candidate's ES probe failed (timeout, non-zero exit, bad JSON)."""
192
203
 
193
204
 
205
+ class ProbeInputError(ValueError):
206
+ """The candidates or options handed to the probe are unusable."""
207
+
208
+
194
209
  def run_es(body):
195
210
  """Run `tl db es - --json` with `body` on stdin; return parsed response.
196
211
 
@@ -324,7 +339,7 @@ def collect_candidates(argv_words):
324
339
  return [ln.strip() for ln in raw.splitlines() if ln.strip()]
325
340
  if isinstance(parsed, list):
326
341
  return parsed
327
- sys.exit("stdin JSON must be a list")
342
+ raise ProbeInputError("stdin JSON must be a list")
328
343
 
329
344
 
330
345
  def normalize(raw_items, default_mode):
@@ -341,7 +356,7 @@ def normalize(raw_items, default_mode):
341
356
  elif "keyword" in item:
342
357
  value, mode = item["keyword"], item.get("mode", default_mode)
343
358
  else:
344
- sys.exit(f"candidate object missing value/sqs/phrase/keyword: {item}")
359
+ raise ProbeInputError(f"candidate object missing value/sqs/phrase/keyword: {item}")
345
360
  value = str(value).strip()
346
361
  label = str(item.get("label", value)).strip() or value
347
362
  else:
@@ -416,90 +431,76 @@ def annotate_recency(row, data, *, level, min_recent_channels, min_share):
416
431
  row["thin"] = below_floor and not stale
417
432
 
418
433
 
419
- def main():
420
- ap = argparse.ArgumentParser(
421
- description="Probe ES for keyword counts + validation samples; drop empty, annotate subsumed."
422
- )
423
- ap.add_argument("keywords", nargs="*", help="Candidates (or pipe a JSON array on stdin)")
424
- ap.add_argument("--operator", choices=["AND", "OR"], default="OR",
425
- help="How the caller will combine survivors downstream (default OR). "
426
- "Echoed in output and drives subsumption annotation (OR only).")
427
- ap.add_argument("--level", choices=["topic", "channel"], default="topic",
428
- help="topic = videos/articles (doc_type:article); "
429
- "channel = whole channels (doc_type:channel). Default topic.")
430
- ap.add_argument("--mode", choices=["phrase", "sqs"], default="phrase",
431
- help="How to interpret plain candidates: phrase match (default) "
432
- "or simple_query_string (Boolean: + | - \"phrase\" () field^boost).")
433
- ap.add_argument("--fields", default=None,
434
- help="Comma-separated ES fields to search (defaults by --level).")
435
- ap.add_argument("--samples", type=int, default=DEFAULT_SAMPLES,
436
- help=f"Sample docs returned per candidate (default {DEFAULT_SAMPLES}, max {MAX_SAMPLES}; 0 = count only).")
437
- ap.add_argument("--content-type", choices=list(CONTENT_TYPES), default="longform",
438
- help="Topic-level video content type filter (default longform). "
439
- "'all' drops the filter (include shorts + live). Ignored at "
440
- "channel level (channel docs have no content type). "
441
- "YouTube-only (format 4) is always enforced.")
442
- ap.add_argument("--since", help="publication_date >= YYYY-MM-DD (topic level only)")
443
- ap.add_argument("--until", help="publication_date <= YYYY-MM-DD (topic level only)")
444
- # Recency validation: headline counts stay ALL-TIME; this layers an extra
445
- # "is the content still active?" signal on the SAME query (no extra round-trip).
446
- ap.add_argument("--no-recency", action="store_true",
447
- help="Skip the recency/active-content check (pure all-time counts).")
448
- ap.add_argument("--recency-months", type=int, default=12,
449
- help="Topic-level recency window in months (default 12). "
450
- "Channel level uses posts_per_90_days>0 (channel docs have no date).")
451
- ap.add_argument("--min-recent-channels", type=int, default=5,
452
- help="STALE floor: fewer than this many recent/active distinct channels (default 5).")
453
- ap.add_argument("--min-recent-share", type=float, default=0.10,
454
- help="Topic STALE share: recent_channels/channels below this AND below the floor → stale (default 0.10).")
455
- ap.add_argument("--min-active-share", type=float, default=0.33,
456
- help="Channel STALE share: active_channels/channels below this AND below the floor → stale (default 0.33).")
457
- args = ap.parse_args()
458
-
459
- if args.recency_months < 1:
460
- sys.exit("--recency-months must be >= 1")
434
+ def check_options(*, fields, level, mode, content_type, operator, since, until, recency_months):
435
+ """Raise ProbeInputError for an unusable option combination (first problem wins)."""
436
+ if level not in COLLAPSE_FIELD:
437
+ raise ProbeInputError(f"level must be one of {sorted(COLLAPSE_FIELD)}, got {level!r}")
438
+ if mode not in ("phrase", "sqs"):
439
+ raise ProbeInputError(f"mode must be phrase or sqs, got {mode!r}")
440
+ if content_type not in CONTENT_TYPES:
441
+ raise ProbeInputError(f"content_type must be one of {list(CONTENT_TYPES)}, got {content_type!r}")
442
+ if operator not in ("AND", "OR"):
443
+ raise ProbeInputError(f"operator must be AND or OR, got {operator!r}")
444
+ if recency_months < 1:
445
+ raise ProbeInputError("--recency-months must be >= 1")
461
446
 
462
447
  # Date windowing only makes sense for articles — channel docs have no
463
448
  # publication_date, so a range filter there silently zeroes every count.
464
- if (args.since or args.until) and args.level != "topic":
465
- sys.exit("--since/--until apply only at --level topic (channel docs have no publication date)")
466
- for flag, val in (("--since", args.since), ("--until", args.until)):
449
+ if (since or until) and level != "topic":
450
+ raise ProbeInputError("--since/--until apply only at --level topic (channel docs have no publication date)")
451
+ for flag, val in (("--since", since), ("--until", until)):
467
452
  if val and not valid_date(val):
468
- sys.exit(f"{flag} must be YYYY-MM-DD, got {val!r}")
453
+ raise ProbeInputError(f"{flag} must be YYYY-MM-DD, got {val!r}")
469
454
 
470
- fields = ([f.strip() for f in args.fields.split(",") if f.strip()] if args.fields
471
- else (TOPIC_FIELDS if args.level == "topic" else CHANNEL_FIELDS))
455
+ if not isinstance(fields, (list, tuple)):
456
+ raise ProbeInputError("fields must be a list of ES field paths")
472
457
  if not fields:
473
- sys.exit("--fields must list at least one ES field")
474
- bad = [f for f in fields if not _FIELD_RE.match(f)]
458
+ raise ProbeInputError("--fields must list at least one ES field")
459
+ bad = [f for f in fields if not (isinstance(f, str) and _FIELD_RE.match(f))]
475
460
  if bad:
476
- sys.exit(f"invalid --fields entries {bad}; use ES field paths, optionally boosted (e.g. title^3)")
461
+ raise ProbeInputError(f"invalid --fields entries {bad}; use ES field paths, optionally boosted (e.g. title^3)")
477
462
 
478
- field_pairs = TOPIC_SAMPLE_FIELDS if args.level == "topic" else CHANNEL_SAMPLE_FIELDS
463
+
464
+ def run_probe(candidates: list, *, run_es: Callable[[dict], dict], fields: list[str], level: str = "topic",
465
+ mode: str = "phrase", samples: int = 5, content_type: str = "longform", since: str | None = None,
466
+ until: str | None = None, operator: str = "OR", recency: bool = True, recency_months: int = 12,
467
+ min_recent_channels: int = 5, min_recent_share: float = 0.10, min_active_share: float = 0.33) -> dict:
468
+ """Probe every candidate through `run_es` and return the object `main()` prints.
469
+
470
+ `candidates` are strings or dicts (what `normalize` accepts). `run_es(body)` takes an
471
+ Elasticsearch body and returns the parsed response; if it raises, that candidate is
472
+ recorded under `failed` and the rest continue. Raises ProbeInputError on bad input.
473
+ """
474
+ check_options(fields=fields, level=level, mode=mode, content_type=content_type, operator=operator,
475
+ since=since, until=until, recency_months=recency_months)
476
+ fields = list(fields)
477
+ if not isinstance(candidates, (list, tuple)):
478
+ raise ProbeInputError("candidates must be a list of strings or objects")
479
+
480
+ field_pairs = TOPIC_SAMPLE_FIELDS if level == "topic" else CHANNEL_SAMPLE_FIELDS
479
481
  source_paths = [p for p, _ in field_pairs]
480
- samples = min(MAX_SAMPLES, max(0, args.samples))
482
+ samples = min(MAX_SAMPLES, max(0, samples))
481
483
 
482
- candidates = normalize(collect_candidates(args.keywords), args.mode)
484
+ candidates = normalize(candidates, mode)
483
485
  if not candidates:
484
- sys.exit("provide at least one candidate (positional args or JSON array on stdin)")
486
+ raise ProbeInputError("provide at least one candidate (positional args or JSON array on stdin)")
485
487
 
486
- recency = not args.no_recency
487
488
  # Topic level needs a date cutoff; channel level uses a non-date sentinel so
488
489
  # build_body emits the posts_per_90_days "active" agg instead.
489
490
  recency_cutoff = None
490
491
  if recency:
491
- recency_cutoff = months_ago_iso(args.recency_months) if args.level == "topic" else "active"
492
+ recency_cutoff = months_ago_iso(recency_months) if level == "topic" else "active"
492
493
 
493
494
  results, failed = [], []
494
495
  for cand in candidates:
495
- body = build_body(cand, fields=fields, level=args.level, samples=samples,
496
- source_paths=source_paths, since=args.since, until=args.until,
497
- recency_cutoff=recency_cutoff, content_type=args.content_type)
496
+ body = build_body(cand, fields=fields, level=level, samples=samples,
497
+ source_paths=source_paths, since=since, until=until,
498
+ recency_cutoff=recency_cutoff, content_type=content_type)
498
499
  try:
499
500
  data = run_es(body)
500
- except ProbeError as exc:
501
- sys.stderr.write(f"probe failed for {cand['value']!r}: {exc}\n")
502
- failed.append({"keyword": cand["value"], "label": cand["label"], "error": str(exc)})
501
+ except Exception as exc:
502
+ failed.append({"keyword": cand["value"], "label": cand["label"],
503
+ "error": str(exc) or type(exc).__name__})
503
504
  continue
504
505
  documents = extract_total(data)
505
506
  channels = extract_distinct(data)
@@ -510,7 +511,7 @@ def main():
510
511
  row = {
511
512
  "keyword": cand["value"],
512
513
  "label": cand["label"],
513
- "count": documents if args.level == "topic" else channels,
514
+ "count": documents if level == "topic" else channels,
514
515
  "documents": documents,
515
516
  "channels": channels,
516
517
  "samples": extract_samples(data, field_pairs) if samples else [],
@@ -518,18 +519,17 @@ def main():
518
519
  "subsumed_by": [],
519
520
  }
520
521
  if recency:
521
- annotate_recency(row, data, level=args.level,
522
- min_recent_channels=args.min_recent_channels,
523
- min_share=(args.min_recent_share if args.level == "topic"
524
- else args.min_active_share))
522
+ annotate_recency(row, data, level=level,
523
+ min_recent_channels=min_recent_channels,
524
+ min_share=(min_recent_share if level == "topic" else min_active_share))
525
525
  results.append(row)
526
526
 
527
- mark_subsumed(results, args.operator)
527
+ mark_subsumed(results, operator)
528
528
 
529
529
  # `count` is 0 exactly when the document total is 0 (a non-empty match always
530
530
  # reaches >=1 distinct channel), so it is the single no-match gate.
531
531
  recency_keys = (("recent_documents", "recent_channels", "recent_channel_share")
532
- if args.level == "topic" else ("active_channels", "active_channel_share"))
532
+ if level == "topic" else ("active_channels", "active_channel_share"))
533
533
  survivors, dropped = [], []
534
534
  for r in results:
535
535
  if r["count"] == 0:
@@ -547,11 +547,11 @@ def main():
547
547
 
548
548
  # Echo the always-on scope so the caller (and user) sees what was filtered.
549
549
  scope = {"format": "youtube"}
550
- if args.level == "topic":
551
- scope["content_type"] = args.content_type
550
+ if level == "topic":
551
+ scope["content_type"] = content_type
552
552
  out = {
553
- "operator": args.operator,
554
- "level": args.level,
553
+ "operator": operator,
554
+ "level": level,
555
555
  "fields": fields,
556
556
  "scope": scope,
557
557
  "keywords": survivors,
@@ -559,10 +559,69 @@ def main():
559
559
  "failed": failed,
560
560
  }
561
561
  if recency:
562
- out["recency"] = ({"months": args.recency_months, "cutoff": recency_cutoff}
563
- if args.level == "topic" else {"signal": "posts_per_90_days>0"})
562
+ out["recency"] = ({"months": recency_months, "cutoff": recency_cutoff}
563
+ if level == "topic" else {"signal": "posts_per_90_days>0"})
564
+ return out
565
+
566
+
567
+ def main():
568
+ ap = argparse.ArgumentParser(
569
+ description="Probe ES for keyword counts + validation samples; drop empty, annotate subsumed."
570
+ )
571
+ ap.add_argument("keywords", nargs="*", help="Candidates (or pipe a JSON array on stdin)")
572
+ ap.add_argument("--operator", choices=["AND", "OR"], default="OR",
573
+ help="How the caller will combine survivors downstream (default OR). "
574
+ "Echoed in output and drives subsumption annotation (OR only).")
575
+ ap.add_argument("--level", choices=["topic", "channel"], default="topic",
576
+ help="topic = videos/articles (doc_type:article); "
577
+ "channel = whole channels (doc_type:channel). Default topic.")
578
+ ap.add_argument("--mode", choices=["phrase", "sqs"], default="phrase",
579
+ help="How to interpret plain candidates: phrase match (default) "
580
+ "or simple_query_string (Boolean: + | - \"phrase\" () field^boost).")
581
+ ap.add_argument("--fields", default=None,
582
+ help="Comma-separated ES fields to search (defaults by --level).")
583
+ ap.add_argument("--samples", type=int, default=DEFAULT_SAMPLES,
584
+ help=f"Sample docs returned per candidate (default {DEFAULT_SAMPLES}, max {MAX_SAMPLES}; 0 = count only).")
585
+ ap.add_argument("--content-type", choices=list(CONTENT_TYPES), default="longform",
586
+ help="Topic-level video content type filter (default longform). "
587
+ "'all' drops the filter (include shorts + live). Ignored at "
588
+ "channel level (channel docs have no content type). "
589
+ "YouTube-only (format 4) is always enforced.")
590
+ ap.add_argument("--since", help="publication_date >= YYYY-MM-DD (topic level only)")
591
+ ap.add_argument("--until", help="publication_date <= YYYY-MM-DD (topic level only)")
592
+ # Recency validation: headline counts stay ALL-TIME; this layers an extra
593
+ # "is the content still active?" signal on the SAME query (no extra round-trip).
594
+ ap.add_argument("--no-recency", action="store_true",
595
+ help="Skip the recency/active-content check (pure all-time counts).")
596
+ ap.add_argument("--recency-months", type=int, default=12,
597
+ help="Topic-level recency window in months (default 12). "
598
+ "Channel level uses posts_per_90_days>0 (channel docs have no date).")
599
+ ap.add_argument("--min-recent-channels", type=int, default=5,
600
+ help="STALE floor: fewer than this many recent/active distinct channels (default 5).")
601
+ ap.add_argument("--min-recent-share", type=float, default=0.10,
602
+ help="Topic STALE share: recent_channels/channels below this AND below the floor → stale (default 0.10).")
603
+ ap.add_argument("--min-active-share", type=float, default=0.33,
604
+ help="Channel STALE share: active_channels/channels below this AND below the floor → stale (default 0.33).")
605
+ args = ap.parse_args()
606
+
607
+ fields = ([f.strip() for f in args.fields.split(",") if f.strip()] if args.fields
608
+ else (TOPIC_FIELDS if args.level == "topic" else CHANNEL_FIELDS))
609
+ options = dict(fields=fields, level=args.level, mode=args.mode, content_type=args.content_type,
610
+ operator=args.operator, since=args.since, until=args.until,
611
+ recency_months=args.recency_months)
612
+ try:
613
+ check_options(**options) # an unusable option outranks a malformed stdin
614
+ candidates = collect_candidates(args.keywords)
615
+ out = run_probe(candidates, run_es=run_es, samples=args.samples, recency=not args.no_recency,
616
+ min_recent_channels=args.min_recent_channels, min_recent_share=args.min_recent_share,
617
+ min_active_share=args.min_active_share, **options)
618
+ except ProbeInputError as exc:
619
+ sys.exit(str(exc))
620
+
621
+ for f in out["failed"]:
622
+ sys.stderr.write(f"probe failed for {f['keyword']!r}: {f['error']}\n")
564
623
  print(json.dumps(out, ensure_ascii=False))
565
- if failed and not results:
624
+ if out["failed"] and not out["keywords"] and not out["dropped"]:
566
625
  sys.exit(1) # every candidate failed — the batch itself is broken
567
626
 
568
627
 
@@ -26,15 +26,56 @@ Usage:
26
26
  python3 select_keywords.py --emit-batch < probe.json > batch.json
27
27
  # ...send batch.json (+ "intent: ..." line) to keyword-relevance-validator → verdict.json...
28
28
  python3 select_keywords.py --apply verdict.json < probe.json
29
+
30
+ Library use: the file is import-safe (standard library only, nothing runs at import),
31
+ so another Python program can load it by path and call `emit_batch(probe)` (the batch
32
+ list) and `apply_verdict_maps(probe, verdict_maps, *, allow_missing=False)` (the
33
+ `--apply` result). `probe` is the parsed `probe.py` output; `verdict_maps` is a list of
34
+ `{i: bool}` maps, one per validator pass. Bad input raises `SelectInputError` (a
35
+ `ValueError`); an incomplete verdict raises `IncompleteVerdictError` with `.missing`, the
36
+ sorted indices without a verdict. With `allow_missing=True` the same gap is reported
37
+ through `warnings.warn` (`MissingVerdictWarning`) instead. Library functions never print,
38
+ read stdin or exit. `apply_verdicts` is the command's own layer (prints, exits).
29
39
  """
40
+ from __future__ import annotations
41
+
30
42
  import argparse
31
43
  import json
32
44
  import sys
45
+ import warnings
33
46
 
34
47
  # Friendly content keys shown to the validator, by probe level.
35
48
  BATCH_CONTENT_KEYS = {"topic": ["title", "summary"], "channel": ["name", "topic"]}
36
49
 
37
50
 
51
+ class SelectInputError(ValueError):
52
+ """The probe output or verdicts handed to the selector are unusable."""
53
+
54
+
55
+ class IncompleteVerdictError(SelectInputError):
56
+ """Some batch samples have no verdict in any pass; `.missing` lists their indices."""
57
+
58
+ def __init__(self, message, missing):
59
+ super().__init__(message)
60
+ self.missing = list(missing)
61
+
62
+
63
+ class MissingVerdictWarning(UserWarning):
64
+ """Verdicts left some batch samples unjudged (allow_missing=True)."""
65
+
66
+
67
+ def check_probe(probe):
68
+ """Raise SelectInputError unless `probe` has the shape probe.py prints."""
69
+ if not isinstance(probe, dict) or not isinstance(probe.get("keywords"), list):
70
+ raise SelectInputError("probe JSON must be an object with a 'keywords' list")
71
+ for kw in probe["keywords"]:
72
+ if not isinstance(kw, dict) or "keyword" not in kw:
73
+ raise SelectInputError("each probe keyword must be an object with a 'keyword'")
74
+ samples = kw.get("samples", [])
75
+ if not isinstance(samples, list) or not all(isinstance(smp, dict) for smp in samples):
76
+ raise SelectInputError("each probe keyword's 'samples' must be a list of objects")
77
+
78
+
38
79
  def load_probe():
39
80
  if sys.stdin.isatty():
40
81
  sys.exit("pipe probe.py output JSON on stdin")
@@ -42,8 +83,10 @@ def load_probe():
42
83
  probe = json.loads(sys.stdin.read())
43
84
  except json.JSONDecodeError as exc:
44
85
  sys.exit(f"invalid probe JSON on stdin: {exc}")
45
- if not isinstance(probe, dict) or not isinstance(probe.get("keywords"), list):
46
- sys.exit("probe JSON must be an object with a 'keywords' list")
86
+ try:
87
+ check_probe(probe)
88
+ except SelectInputError as exc:
89
+ sys.exit(str(exc))
47
90
  return probe
48
91
 
49
92
 
@@ -59,6 +102,8 @@ def build_index(probe):
59
102
 
60
103
 
61
104
  def emit_batch(probe):
105
+ """The flat validation batch the keyword-relevance-validator expects (what --emit-batch prints)."""
106
+ check_probe(probe)
62
107
  level = probe.get("level", "topic")
63
108
  keys = BATCH_CONTENT_KEYS.get(level, BATCH_CONTENT_KEYS["topic"])
64
109
  batch = []
@@ -67,7 +112,7 @@ def emit_batch(probe):
67
112
  for k in keys:
68
113
  item[k] = sample.get(k)
69
114
  batch.append(item)
70
- print(json.dumps(batch, ensure_ascii=False))
115
+ return batch
71
116
 
72
117
 
73
118
  def load_verdicts(paths):
@@ -106,30 +151,47 @@ def check_completeness(index, verdict_maps, allow_missing):
106
151
  """Every batch sample must have a verdict in at least one pass.
107
152
 
108
153
  Cheap validators silently drop the tail of a long list, so a missing `i`
109
- usually means truncation, not judgement. Default: fail loudly and list
110
- the missing indices so the caller re-sends just those samples to a fresh
111
- validator run. `--allow-missing` downgrades to a stderr warning (missing
112
- samples then count as not-relevant, which biases toward dropping).
154
+ usually means truncation, not judgement. Default: raise IncompleteVerdictError
155
+ listing the missing indices so the caller re-sends just those samples to a
156
+ fresh validator run. With `allow_missing` the gap is returned as warning text
157
+ instead (missing samples then count as not-relevant, which biases toward
158
+ dropping). Returns None when every sample has a verdict.
113
159
  """
114
160
  covered = set()
115
161
  for vm in verdict_maps:
116
162
  covered.update(vm)
117
163
  missing = sorted(i for i, _, _ in index if i not in covered)
118
164
  if not missing:
119
- return
165
+ return None
120
166
  msg = (f"{len(missing)} of {len(index)} batch samples have no verdict "
121
167
  f"(missing i: {missing[:20]}{'…' if len(missing) > 20 else ''}) — "
122
168
  f"the validator likely truncated; re-send those samples and merge.")
123
169
  if allow_missing:
124
- sys.stderr.write(f"warning: {msg}\n")
125
- else:
126
- sys.exit(msg + " (or pass --allow-missing to treat them as not relevant)")
170
+ return msg
171
+ raise IncompleteVerdictError(msg, missing)
127
172
 
128
173
 
129
- def apply_verdicts(probe, verdict_maps, allow_missing=False):
174
+ def apply_verdict_maps(probe, verdict_maps, *, allow_missing=False):
175
+ """Decide each keyword from the verdicts and return what --apply prints.
176
+
177
+ `verdict_maps` is a list of {i: bool} maps, one per validator pass. Raises
178
+ SelectInputError / IncompleteVerdictError; with `allow_missing` a gap is
179
+ announced through `warnings.warn` (MissingVerdictWarning) instead.
180
+ """
181
+ out, warning = _apply(probe, verdict_maps, allow_missing)
182
+ if warning:
183
+ warnings.warn(warning, MissingVerdictWarning, stacklevel=2)
184
+ return out
185
+
186
+
187
+ def _apply(probe, verdict_maps, allow_missing):
188
+ """The decision itself: (result, gap-warning text or None)."""
189
+ check_probe(probe)
190
+ if not isinstance(verdict_maps, (list, tuple)) or not all(isinstance(vm, dict) for vm in verdict_maps):
191
+ raise SelectInputError("verdict_maps must be a list of {i: relevant} maps")
130
192
  index = build_index(probe)
131
193
  level = probe.get("level", "topic")
132
- check_completeness(index, verdict_maps, allow_missing)
194
+ warning = check_completeness(index, verdict_maps, allow_missing)
133
195
  finals = {i: relevant_at(i, verdict_maps) for i, _, _ in index}
134
196
 
135
197
  per_kw = {} # keyword -> {"relevant": n, "total": n, "samples":[(rel, sample)]}
@@ -188,6 +250,19 @@ def apply_verdicts(probe, verdict_maps, allow_missing=False):
188
250
  }
189
251
  if level != "channel":
190
252
  out["candidate_videos"] = candidate_videos
253
+ return out, warning
254
+
255
+
256
+ def apply_verdicts(probe, verdict_maps, allow_missing=False):
257
+ """The command's --apply layer: print the result, exit on an incomplete verdict, warn on stderr."""
258
+ try:
259
+ out, warning = _apply(probe, verdict_maps, allow_missing)
260
+ except IncompleteVerdictError as exc:
261
+ sys.exit(f"{exc} (or pass --allow-missing to treat them as not relevant)")
262
+ except SelectInputError as exc:
263
+ sys.exit(str(exc))
264
+ if warning:
265
+ sys.stderr.write(f"warning: {warning}\n")
191
266
  print(json.dumps(out, ensure_ascii=False, indent=2))
192
267
 
193
268
 
@@ -203,7 +278,7 @@ def main():
203
278
 
204
279
  probe = load_probe()
205
280
  if args.emit_batch:
206
- emit_batch(probe)
281
+ print(json.dumps(emit_batch(probe), ensure_ascii=False))
207
282
  else:
208
283
  apply_verdicts(probe, load_verdicts(args.apply), allow_missing=args.allow_missing)
209
284
 
@@ -41,7 +41,7 @@
41
41
  {"name": "Avg. Comments", "backend_code": "comments_avg", "sortability": "both", "description": "Average comments per YouTube video."},
42
42
  {"name": "Median Evergreenness", "backend_code": "evergreenness_median", "sortability": "both", "description": "Median evergreenness for the channel."},
43
43
  {"name": "Avg. Evergreenness", "backend_code": "evergreenness_avg", "sortability": "both", "description": "Average evergreenness for the channel."},
44
- {"name": "Latest AdSpot Price", "backend_code": "latest_adspot_price", "sortability": "both", "description": "Price from the latest active AdSpot."},
44
+ {"name": "Latest AdSpot Price", "backend_code": "latest_adspot_price", "sortability": "both", "description": "Price from the latest preferred AdSpot."},
45
45
  {"name": "Subscribers", "backend_code": "subscribers", "sortability": "both", "description": "Channel subscriber count."},
46
46
  {"name": "Projected Views", "backend_code": "projected_views", "sortability": "both", "description": "Projected views for the channel's next video."},
47
47
  {"name": "Trend", "backend_code": "trend", "sortability": "both", "description": "Growth trend score — higher means faster-growing channel."}
@@ -71,7 +71,7 @@ def show_cmd(
71
71
  csv_output: bool = typer.Option(False, "--csv", help="CSV output (flattens adspots: one row per adspot)"),
72
72
  toon_output: bool = typer.Option(False, "--toon", help="TOON output (token-efficient for LLMs)"),
73
73
  ) -> None:
74
- """Show channel detail by ID or name (includes active adspots).
74
+ """Show channel detail by ID or name (includes preferred adspots).
75
75
 
76
76
  Accepts either a numeric channel ID or a partial name. Names that
77
77
  match more than one active channel return a 400 with the candidate