thoughtleaders-cli 0.9.22__py3-none-any.whl → 0.9.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {thoughtleaders_cli-0.9.22.dist-info → thoughtleaders_cli-0.9.24.dist-info}/METADATA +1 -1
- {thoughtleaders_cli-0.9.22.dist-info → thoughtleaders_cli-0.9.24.dist-info}/RECORD +17 -17
- tl_cli/__init__.py +1 -1
- tl_cli/_plugin/.claude-plugin/plugin.json +1 -1
- tl_cli/_plugin/skills/tl/SKILL.md +2 -2
- tl_cli/_plugin/skills/tl/references/business-glossary.md +2 -2
- tl_cli/_plugin/skills/tl/references/postgres-schema.md +3 -2
- tl_cli/_plugin/skills/tl-keyword-research/SKILL.md +79 -0
- tl_cli/_plugin/skills/tl-keyword-research/scripts/build_report.py +48 -25
- tl_cli/_plugin/skills/tl-keyword-research/scripts/expand_entities.py +50 -6
- tl_cli/_plugin/skills/tl-keyword-research/scripts/probe.py +141 -82
- tl_cli/_plugin/skills/tl-keyword-research/scripts/select_keywords.py +89 -14
- tl_cli/_plugin/skills/tl-save-report/references/sortable_columns.json +1 -1
- tl_cli/commands/channels.py +1 -1
- {thoughtleaders_cli-0.9.22.dist-info → thoughtleaders_cli-0.9.24.dist-info}/WHEEL +0 -0
- {thoughtleaders_cli-0.9.22.dist-info → thoughtleaders_cli-0.9.24.dist-info}/entry_points.txt +0 -0
- {thoughtleaders_cli-0.9.22.dist-info → thoughtleaders_cli-0.9.24.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: thoughtleaders-cli
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.24
|
|
4
4
|
Summary: ThoughtLeaders CLI — query sponsorship data, channels, brands, and intelligence
|
|
5
5
|
Project-URL: Homepage, https://thoughtleaders.io
|
|
6
6
|
Project-URL: Repository, https://github.com/ThoughtLeaders-io/thoughtleaders-cli
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
tl_cli/__init__.py,sha256=
|
|
1
|
+
tl_cli/__init__.py,sha256=Fr4WPfFPO4tfJasNz2cfjOo8stYGSDkrJhJ8azhBL5o,113
|
|
2
2
|
tl_cli/__main__.py,sha256=-FCBeOBi5Zi0hhZI3FWRilisl2DHxKeszgkY0r9VLR4,444
|
|
3
3
|
tl_cli/_completions.py,sha256=kOyEUqC26vbYvyXWi513WX8fF73qQLR5WWuRSe_wqyk,164
|
|
4
4
|
tl_cli/_typer_utils.py,sha256=ZiZsCVmEznPvBw-dYbr3tu3zWZ0iN6kjoQmK3gMqD28,860
|
|
@@ -24,7 +24,7 @@ tl_cli/commands/balance.py,sha256=s5uH0ijahT2gfgOKzOFxYxHyRBq50mT8zpNCY6KEsBY,37
|
|
|
24
24
|
tl_cli/commands/brands.py,sha256=uogOfAdEbpjqiGlv4L2T1UC-Iz10G6fkRhVsDBOjRIY,15340
|
|
25
25
|
tl_cli/commands/bulk_import.py,sha256=d4y1k_lD52LPJcCqXxEmyHIqcIwomZgbjqs1_QxPjeQ,4536
|
|
26
26
|
tl_cli/commands/changelog.py,sha256=D1PtDdHpawTlWqUHjKzVmv9yXLSU915UVmI3dZzEwyA,4241
|
|
27
|
-
tl_cli/commands/channels.py,sha256=
|
|
27
|
+
tl_cli/commands/channels.py,sha256=ib1kTYNr0MjWVEbWLUmONX3-kyj0JLtwLGpTCxHwjHU,20409
|
|
28
28
|
tl_cli/commands/credits.py,sha256=2xCht2e420LmaFBKNdKoMz8GlTh31qSWSlJAnVzoZic,7308
|
|
29
29
|
tl_cli/commands/db.py,sha256=RcuGYzmTs5KMk6RgYao3oV-j883dG7IonTQclZg7UZs,9148
|
|
30
30
|
tl_cli/commands/deals.py,sha256=ZK9yneInsC6DXoCPS65oyLoVR0eRW1xdRlEN7oRp1pc,2174
|
|
@@ -47,7 +47,7 @@ tl_cli/commands/workflows.py,sha256=ctduYa--Md6zbs8kQwO6mSyR9o1KB1UpNOaANuupDfU,
|
|
|
47
47
|
tl_cli/output/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
48
48
|
tl_cli/output/formatter.py,sha256=itNyHcDfRbksJgJ58JLtwJIrlHjqQj9gmK6uc3d9bSM,25692
|
|
49
49
|
tl_cli/_plugin/.claude-plugin/marketplace.json,sha256=l56PMmyjfGXNGlV30wRyOAe74B6gJNCVNCxgsBbSNxc,446
|
|
50
|
-
tl_cli/_plugin/.claude-plugin/plugin.json,sha256=
|
|
50
|
+
tl_cli/_plugin/.claude-plugin/plugin.json,sha256=RE1yHOU6LJMeA2u0DwL08urULLGRQ0XGFxq6srEvJD4,467
|
|
51
51
|
tl_cli/_plugin/agents/cast-sheet.md,sha256=0owR8XIjR4tbKuK4t3WruiJAjGQ8se2cnOGwzigs4Lk,1224
|
|
52
52
|
tl_cli/_plugin/agents/gem-classifier.md,sha256=s9kkbhUc-JhJ_be9z6LNaojykQSV16zrOd-QVaKNwxc,1250
|
|
53
53
|
tl_cli/_plugin/agents/keyword-context-classifier.md,sha256=8Pk6WqJKuF-ERkBh54Aa-9t60CvjdOIH0RurFlG-T7I,4325
|
|
@@ -65,11 +65,11 @@ tl_cli/_plugin/skills/connector-impact/SKILL.md,sha256=K4mozNpRSLjTSICRTX8LihnKI
|
|
|
65
65
|
tl_cli/_plugin/skills/connector-impact/references/help.md,sha256=Wd8X35OwYAswj-xoFZPnCWctwFIx9EXNZ511FDB2I5c,5831
|
|
66
66
|
tl_cli/_plugin/skills/connector-impact/references/impact-schema.md,sha256=dlAtA6OwkqgXojaItnaQBEIXVxLoVuavuWsPtQ4Rc_0,10155
|
|
67
67
|
tl_cli/_plugin/skills/connector-impact/references/impact-to-tl-mapping.md,sha256=58IJ5r1I0aEmIETFm81vMAbd1NxVMol8-fqqvHG3qXY,22122
|
|
68
|
-
tl_cli/_plugin/skills/tl/SKILL.md,sha256=
|
|
69
|
-
tl_cli/_plugin/skills/tl/references/business-glossary.md,sha256=
|
|
68
|
+
tl_cli/_plugin/skills/tl/SKILL.md,sha256=C9XAkgeQUB-Or8-CSbZPVwoh9JS184bK0gj1y7nNiJ8,72757
|
|
69
|
+
tl_cli/_plugin/skills/tl/references/business-glossary.md,sha256=DkM-3DGnI_Dvr8fsB9B00-5erXv9r94umFCMVBGOxEs,20214
|
|
70
70
|
tl_cli/_plugin/skills/tl/references/elasticsearch-schema.md,sha256=zjuHmJRooTTZvu2GfoQxG8xmRcExcKa33K1jipLAShI,25222
|
|
71
71
|
tl_cli/_plugin/skills/tl/references/firebolt-schema.md,sha256=yf_elP_qXQePtMJNMshoVUOrI_3CXezF0NOoiYTbMNU,11335
|
|
72
|
-
tl_cli/_plugin/skills/tl/references/postgres-schema.md,sha256=
|
|
72
|
+
tl_cli/_plugin/skills/tl/references/postgres-schema.md,sha256=q0f264d1KhE96IwmVp7u-GX2N3tlanUiSIIPlqemIfo,30614
|
|
73
73
|
tl_cli/_plugin/skills/tl-brand-creator-connection/SKILL.md,sha256=6__d4Z1_Mdb8m9nMKxU2wAYxAfu_3o_ucqn-0nXzqJs,35615
|
|
74
74
|
tl_cli/_plugin/skills/tl-brand-creator-connection/references/cast-sheet.md,sha256=Y1Yj0D36tTGldn6uILoDOYwdMyjAp1oT9P-nUNgSyLg,4393
|
|
75
75
|
tl_cli/_plugin/skills/tl-brand-creator-connection/references/creator-brief-template.md,sha256=mtFNSCPfj2fGrKtMX2g-Q1UNHYs59H4fARz320EutJA,10901
|
|
@@ -122,16 +122,16 @@ tl_cli/_plugin/skills/tl-create-workflow/references/creating-in-app.md,sha256=mJ
|
|
|
122
122
|
tl_cli/_plugin/skills/tl-create-workflow/references/methodology.md,sha256=oXth-akS6QD7GmSDQRMaenAViTC5tlRgXIq_5uXVQQg,3847
|
|
123
123
|
tl_cli/_plugin/skills/tl-create-workflow/references/pitfalls.md,sha256=6XBbe7T3o9PQcEs5FGNZ3n6xqVz0ZoNoFttJ9nNDjBU,3289
|
|
124
124
|
tl_cli/_plugin/skills/tl-create-workflow/references/workflow-model.md,sha256=39cNg6AaSjb5K54neBkkn_RVJa7g9UnfZ_pVSzUsjDI,4591
|
|
125
|
-
tl_cli/_plugin/skills/tl-keyword-research/SKILL.md,sha256=
|
|
125
|
+
tl_cli/_plugin/skills/tl-keyword-research/SKILL.md,sha256=9KtLmWzPr93_i1dqrPnFx5yLC58EVoFGpIZJRG1kf6Y,41630
|
|
126
126
|
tl_cli/_plugin/skills/tl-keyword-research/references/elasticsearch-content-search.md,sha256=fsXbpI3DPtkOnPEquB11Efur665o963RDN4-WIJCRJs,24263
|
|
127
127
|
tl_cli/_plugin/skills/tl-keyword-research/references/help.md,sha256=yTmoU-Z6rPDjprbDa09nePoON0jnRSh_FVc7lqu1XGs,6802
|
|
128
|
-
tl_cli/_plugin/skills/tl-keyword-research/scripts/build_report.py,sha256=
|
|
129
|
-
tl_cli/_plugin/skills/tl-keyword-research/scripts/expand_entities.py,sha256=
|
|
128
|
+
tl_cli/_plugin/skills/tl-keyword-research/scripts/build_report.py,sha256=T_3xQ7vo_wdyZD5ObHrH5Lo_T-TcmBDygvXpfK-4P5I,23149
|
|
129
|
+
tl_cli/_plugin/skills/tl-keyword-research/scripts/expand_entities.py,sha256=r60FsvQiOsrCHG9xM1sGbzcN4kQTjWHSoiprVy61Ps4,12940
|
|
130
130
|
tl_cli/_plugin/skills/tl-keyword-research/scripts/fetch_context.py,sha256=dn4n-WPjz4lib42hrDxrTN_xwqy1GZDpURE-IE6OKkg,6877
|
|
131
|
-
tl_cli/_plugin/skills/tl-keyword-research/scripts/probe.py,sha256=
|
|
131
|
+
tl_cli/_plugin/skills/tl-keyword-research/scripts/probe.py,sha256=WiDdo39Wd0CXau2kn4Z5l__8dr9Spr2TkN3OCRPdj8o,31755
|
|
132
132
|
tl_cli/_plugin/skills/tl-keyword-research/scripts/search_channels.py,sha256=qoe70bfpSWujO12eFP-OnfrqwbI7FgWTP_9aWavw3-U,23189
|
|
133
133
|
tl_cli/_plugin/skills/tl-keyword-research/scripts/search_videos.py,sha256=Q_fYr1BzrMr5nh9wC1ruBJuIQFzNBkkBnYNMmsSEnc4,14568
|
|
134
|
-
tl_cli/_plugin/skills/tl-keyword-research/scripts/select_keywords.py,sha256=
|
|
134
|
+
tl_cli/_plugin/skills/tl-keyword-research/scripts/select_keywords.py,sha256=yQagYIN_Ql7AS-A02SwrUwOGEMNTyyms8hQYCcDZmPo,12791
|
|
135
135
|
tl_cli/_plugin/skills/tl-save-report/SKILL.md,sha256=Sf1zIbLaJnCq2-IctIJrQDWWsTXJOTkkNzg4poc-wdo,37550
|
|
136
136
|
tl_cli/_plugin/skills/tl-save-report/references/columns_brands.md,sha256=H5G308GOzMWvWw6smkPGdb2CBuuxkrojCpJgWS9X4sA,3559
|
|
137
137
|
tl_cli/_plugin/skills/tl-save-report/references/columns_channels.md,sha256=g57ET_VI611lB7fVrGwYFvTDqMkNB63OmmfrnbMZF1I,4684
|
|
@@ -140,7 +140,7 @@ tl_cli/_plugin/skills/tl-save-report/references/columns_sponsorships.md,sha256=I
|
|
|
140
140
|
tl_cli/_plugin/skills/tl-save-report/references/intelligence_filterset_schema.json,sha256=M_gUX6qDLUrAEcocYjU6ta6UaQS1kUSZpogCZzdqCTk,18140
|
|
141
141
|
tl_cli/_plugin/skills/tl-save-report/references/intelligence_widget_schema.json,sha256=cxdlzmkt5WoI20zGsq_4qENTc6Ral9_vI5YTx2yzf-g,11503
|
|
142
142
|
tl_cli/_plugin/skills/tl-save-report/references/report_glossary.md,sha256=YkUxEkUVui9HVC45BYBoyKVuyc_sLmf2GkSAEksmzW0,12165
|
|
143
|
-
tl_cli/_plugin/skills/tl-save-report/references/sortable_columns.json,sha256=
|
|
143
|
+
tl_cli/_plugin/skills/tl-save-report/references/sortable_columns.json,sha256=Q8SYRqoMKMvu0r6qMWlSkmGdI_r5_Q9PwWhidqLFPig,7574
|
|
144
144
|
tl_cli/_plugin/skills/tl-save-report/references/sponsorship_filterset_schema.json,sha256=7bS5G19A6JxOFyx-QoueURBoOf23FxCylwLeqgh8aKk,11381
|
|
145
145
|
tl_cli/_plugin/skills/tl-save-report/references/sponsorship_widget_schema.json,sha256=PXEhZzRTNl94fFHpsBuo4Gop-Mdveg6GCaIm-ZGhBns,9587
|
|
146
146
|
tl_cli/_plugin/skills/tl-save-report/references/widgets.md,sha256=LrqRyvZHOXVF15FglkeYh6io7f3cYmZEpH00WhEJWKw,9881
|
|
@@ -148,8 +148,8 @@ tl_cli/_plugin/skills/tl-top-partnerships/SKILL.md,sha256=jOMMr40XRnAZv-oRLeyKJT
|
|
|
148
148
|
tl_cli/_plugin/skills/tl-top-partnerships/scripts/top_partnerships.py,sha256=OqhoyvFe1zrtrStChcam7oPYgFnSbGHC3WAgLhLIC3w,13738
|
|
149
149
|
tl_cli/_plugin/skills/tl-views-guarantee/SKILL.md,sha256=IH7q1WJDWri9TWJMiga1FMGJO_GKSbWwaDS6CVNZ9c0,9270
|
|
150
150
|
tl_cli/_plugin/skills/tl-views-guarantee/scripts/vg.py,sha256=5FucKm0WXa16btyUKMyGpdRQ5euVhzKMSoTonuwlm5I,16444
|
|
151
|
-
thoughtleaders_cli-0.9.
|
|
152
|
-
thoughtleaders_cli-0.9.
|
|
153
|
-
thoughtleaders_cli-0.9.
|
|
154
|
-
thoughtleaders_cli-0.9.
|
|
155
|
-
thoughtleaders_cli-0.9.
|
|
151
|
+
thoughtleaders_cli-0.9.24.dist-info/METADATA,sha256=ZEkuSp3hTzzSb7YHv8TR3OeGYUpI6f7ynkpRrBbsH9E,22294
|
|
152
|
+
thoughtleaders_cli-0.9.24.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
153
|
+
thoughtleaders_cli-0.9.24.dist-info/entry_points.txt,sha256=umZp-1BkGkHDG0bNZXpTXrjwW0HGf9IDFN40eAWuuvg,39
|
|
154
|
+
thoughtleaders_cli-0.9.24.dist-info/licenses/LICENSE,sha256=RUfdfLsn6jygiyrnnVUHt6r4IPwr2rbDm9Kixgtu8fo,1071
|
|
155
|
+
thoughtleaders_cli-0.9.24.dist-info/RECORD,,
|
tl_cli/__init__.py
CHANGED
|
@@ -95,7 +95,7 @@ Other key concepts:
|
|
|
95
95
|
- Org scoping for sponsorships is profile-mediated: a sponsorship belongs to your org if **either** `advertiser_profile.organization` (brand side) **or** the adspot seller's organization — `ad_spot.publisher` → that user's profile → `organization` (publisher side) — matches yours. Representing the channel does not surface its deals.
|
|
96
96
|
- **MSN** (Media Selling Network) — the ~12k YouTube channels that have opted in to receive sponsorship offers. A channels is in the MSN group if the `channel.media_selling_network_join_date` field is not null.
|
|
97
97
|
- **MBN** (Media Buying Network) — the brand-side counterpart to MSN: brand profiles that have opted in to receive proposed sponsorships. A profile is in the MBN group if the `profile.media_buying_network_join_date` field is not null.
|
|
98
|
-
- **TPP** (ThoughtLeaders Partner Program, a.k.a. "TL channels") — the ~170 channels TL has the closest working relationship with. A channel is in the TPP group if `channel.is_tpp` is True. **Prefer TPP channels when booking**: they respond fastest, are the easiest to close, and don't need an outreach round-trip — treat them as immediately bookable. TPP is a strict subset of MSN, so the same booking rules (one
|
|
98
|
+
- **TPP** (ThoughtLeaders Partner Program, a.k.a. "TL channels") — the ~170 channels TL has the closest working relationship with. A channel is in the TPP group if `channel.is_tpp` is True. **Prefer TPP channels when booking**: they respond fastest, are the easiest to close, and don't need an outreach round-trip — treat them as immediately bookable. TPP is a strict subset of MSN, so the same booking rules (one preferred mention adspot, etc.) apply.
|
|
99
99
|
- **`demographics_updated_at`** (on channels) — If non-null, the channel has demographics screenshots on file. If null, no demographics screenshots have been uploaded. Use this to check whether a channel has demographics data from screenshots.
|
|
100
100
|
- **`subscribers`** (on channels) — the channel's subscriber count.
|
|
101
101
|
- **`projected_views`** (on channels) — projected views per video on that channel. Forward-looking estimate. May be null when not yet computed. ⚠️ NOT actual views and NOT ad-industry "impressions" (ads served).
|
|
@@ -103,7 +103,7 @@ Other key concepts:
|
|
|
103
103
|
- **`views_guarantee`** (on sponsorships) — projected/guaranteed impressions for the sponsorship. Numeric.
|
|
104
104
|
- **Sponsorship detail fields** (returned by `tl sponsorships show <id> --json`) — the detail payload includes `integration` (raw int), `publish_count`, `common_name`, `outreach_email`, nested `publisher` (`first_name`, `last_name`, `email`), nested `brand_contact` (`first_name`, `last_name`, `email`), and `brand.organization_name`. Use these when generating IOs, contracts, or outreach.
|
|
105
105
|
- **CPM** has two distinct meanings depending on level — pick the one the user actually wants:
|
|
106
|
-
- **Channel CPM** = `(adspot.price / channel.projected_views) * 1000` — projected price per thousand projected views. Used for pricing decisions **before** a sponsorship is sold. Available for channels with
|
|
106
|
+
- **Channel CPM** = `(adspot.price / channel.projected_views) * 1000` — projected price per thousand projected views. Used for pricing decisions **before** a sponsorship is sold. Available for channels with preferred adspots via `tl channels show <channel_id>`.
|
|
107
107
|
- **Sponsorship CPM** = calculated in either of two ways: if `views` is present, then CPM is `(sponsorship.price / sponsorship.views) × 1000`, meaning realized cost per thousand actual views, computed post-publication. If `views` is null, Compute from the sponsorship's `price` and the channel's `projected_views` fields.
|
|
108
108
|
- Where possible, calculate the correct CPM in a SQL expression.
|
|
109
109
|
- **Sponsorship dates** — each sponsorship has four distinct dates, useful for different queries:
|
|
@@ -56,7 +56,7 @@ Maps business terms to database concepts.
|
|
|
56
56
|
- A channel can have **multiple ad spots** because different people sell the same channel (talent manager, direct, multiple agencies)
|
|
57
57
|
- Ad spots are the **catalogue** — adlinks are **instances** of catalogue items
|
|
58
58
|
- Price/cost on adspot = list/catalog values; price/cost on adlink = actual deal values
|
|
59
|
-
- **Only one
|
|
59
|
+
- **Only one preferred adspot with integration=mention per channel at any time** (MSN rule)
|
|
60
60
|
|
|
61
61
|
## Ownership & Accountability
|
|
62
62
|
|
|
@@ -79,7 +79,7 @@ Maps business terms to database concepts.
|
|
|
79
79
|
- Key data: **who is the contact** to buy the ad from
|
|
80
80
|
- `thoughtleaders_channel.media_selling_network_join_date` = when channel joined MSN
|
|
81
81
|
- `thoughtleaders_channel.is_tpp` = TPP channel — TL's closest partners, a strict subset of MSN. **Prefer when booking**: fastest response, easiest to close. Not a gating flag; not a process barrier.
|
|
82
|
-
- **Rule:** Only one
|
|
82
|
+
- **Rule:** Only one preferred adspot with `integration=mention` per channel at any time
|
|
83
83
|
- MSN quality depends on having current, accurate contact info
|
|
84
84
|
|
|
85
85
|
## Channels & Audience
|
|
@@ -165,8 +165,9 @@ A channel can have multiple adspots (different sellers: talent manager, direct,
|
|
|
165
165
|
| `channel_id` | int FK | → `thoughtleaders_channel.id` |
|
|
166
166
|
| `price` | numeric | List/catalogue price |
|
|
167
167
|
| `cost` | numeric | List/catalogue cost |
|
|
168
|
-
| `integration` | int | 1=YouTube Mentions (live reads). Only one
|
|
169
|
-
| `
|
|
168
|
+
| `integration` | int | 1=YouTube Mentions (live reads). Only one preferred mention-type adspot per channel. |
|
|
169
|
+
| `is_preferred` | boolean | The channel's preferred spot, the one new proposals are booked on |
|
|
170
|
+
| `is_active` | boolean | False = deactivated, unusable, kept only for its AdLinks. Client sandboxes never show deactivated spots or their AdLinks |
|
|
170
171
|
| `publisher_id` | int FK | → `auth_user.id` (NOT `thoughtleaders_profile.id` — see gotcha below) |
|
|
171
172
|
|
|
172
173
|
### Key columns for the `thoughtleaders_channel` table
|
|
@@ -23,6 +23,33 @@ description: |
|
|
|
23
23
|
common is X". Also invoke for HELP asks about this skill — "help", "how
|
|
24
24
|
does keyword research work", "what are my options", "describe this skill" —
|
|
25
25
|
answered free from the built-in guide, no queries run.
|
|
26
|
+
output-schema:
|
|
27
|
+
type: object
|
|
28
|
+
required: [keyword_groups, keyword_operator, assumptions]
|
|
29
|
+
properties:
|
|
30
|
+
keyword_groups:
|
|
31
|
+
type: array
|
|
32
|
+
items:
|
|
33
|
+
type: object
|
|
34
|
+
required: [text]
|
|
35
|
+
properties:
|
|
36
|
+
text: {type: string}
|
|
37
|
+
content_fields:
|
|
38
|
+
type: array
|
|
39
|
+
items: {type: string}
|
|
40
|
+
exclude: {type: boolean}
|
|
41
|
+
keyword_operator: {type: string, enum: [OR, AND]}
|
|
42
|
+
assumptions:
|
|
43
|
+
type: array
|
|
44
|
+
items: {type: string}
|
|
45
|
+
dropped:
|
|
46
|
+
type: array
|
|
47
|
+
items:
|
|
48
|
+
type: object
|
|
49
|
+
required: [text, reason]
|
|
50
|
+
properties:
|
|
51
|
+
text: {type: string}
|
|
52
|
+
reason: {type: string}
|
|
26
53
|
---
|
|
27
54
|
|
|
28
55
|
# tl-keyword-research — topic → validated filter set + the results you choose
|
|
@@ -69,6 +96,33 @@ verbatim via `search_channels.py --group` / `search_videos.py --group`.
|
|
|
69
96
|
|
|
70
97
|
`<SKILL_DIR>` below is this skill's directory (the one holding `SKILL.md`).
|
|
71
98
|
|
|
99
|
+
## On the ThoughtLeaders platform
|
|
100
|
+
|
|
101
|
+
The same skill also runs on the ThoughtLeaders platform, when a user starts
|
|
102
|
+
keyword research from their dashboard. You are on the platform when the tools
|
|
103
|
+
`kw_probe` and `kw_validate` are available. The method in this file applies
|
|
104
|
+
unchanged, with these differences:
|
|
105
|
+
|
|
106
|
+
- **No shell and no sub-agents.** Stages that run a script or an agent carry
|
|
107
|
+
an *On the platform* note naming the tool to use instead.
|
|
108
|
+
- **Nobody to ask.** The task gives you the path (quick or deep) and the
|
|
109
|
+
deliverable. Make every scope call yourself and record it in `assumptions`.
|
|
110
|
+
Run the deep path autonomously: no checkpoint after round 3.
|
|
111
|
+
- **Titles and summaries only.** Search the `title` and `summary` fields,
|
|
112
|
+
never `transcript`.
|
|
113
|
+
- **Stop after Stage 4.** The platform builds the trend data, channel tiers,
|
|
114
|
+
report link and previews from your final groups, so skip Stage 5 and the
|
|
115
|
+
scripts in Stage 6.
|
|
116
|
+
- **Decide, don't count.** Finish with `submit_result`, shaped by this
|
|
117
|
+
skill's `output-schema`: the final keyword groups, the operator, your
|
|
118
|
+
assumptions, and the groups you dropped with a reason. Never put counts or
|
|
119
|
+
channel figures in the result.
|
|
120
|
+
- **Narrate with `report_progress`.** Call it at the start of each stage with
|
|
121
|
+
the stage name exactly as the pipeline table writes it (`Set up`, `Expand`,
|
|
122
|
+
`Probe`, `Validate keywords`, `Refine`), instead of messaging the user.
|
|
123
|
+
- **References.** Read `references/elasticsearch-content-search.md` with
|
|
124
|
+
`read_skill_file`.
|
|
125
|
+
|
|
72
126
|
## When to invoke / skip
|
|
73
127
|
|
|
74
128
|
Invoke when the user wants **videos or channels by content** (topics, concepts,
|
|
@@ -234,6 +288,10 @@ index's samples, and every web-sourced name earns its place through the same
|
|
|
234
288
|
probe + validation as any other candidate (no benefit of the doubt; a
|
|
235
289
|
hallucinated name probes to `count: 0` and drops).
|
|
236
290
|
|
|
291
|
+
*On the platform:* use `web_search` (at most two searches) in place of the
|
|
292
|
+
`keyword-entity-resolver` agent, then add each name and its spelling variants
|
|
293
|
+
to your candidates yourself, following Stage 1's rules.
|
|
294
|
+
|
|
237
295
|
### Stage 1 — Expand: go wide, go deep, go creative
|
|
238
296
|
|
|
239
297
|
A shallow synonym list is the #1 failure mode. Coverage is won or lost here.
|
|
@@ -271,6 +329,11 @@ Rules:
|
|
|
271
329
|
|
|
272
330
|
### Stage 2 — Probe (`probe.py`)
|
|
273
331
|
|
|
332
|
+
*On the platform:* call `kw_probe` with the candidates and the same options
|
|
333
|
+
`probe.py` takes (`level`, `mode`, `fields`, `samples`, `content_type`,
|
|
334
|
+
`since`, `until`). It returns the counts; the samples stay with the platform
|
|
335
|
+
for `kw_validate`.
|
|
336
|
+
|
|
274
337
|
One ES query per candidate → counts + samples for validation:
|
|
275
338
|
|
|
276
339
|
```bash
|
|
@@ -322,6 +385,11 @@ surviving variants before delivery.
|
|
|
322
385
|
This fixes "the word is there but the topic isn't" — the step that separates
|
|
323
386
|
this skill from a free YouTube search.
|
|
324
387
|
|
|
388
|
+
*On the platform:* call `kw_validate` on the probe with the verbatim intent. It
|
|
389
|
+
runs the relevance check and the strict-majority rule, and returns the kept
|
|
390
|
+
and dropped keywords. Scope calls are yours: decide them and record them in
|
|
391
|
+
`assumptions`.
|
|
392
|
+
|
|
325
393
|
**Inline (≤ ~15 candidates):** read each keyword's `samples` against the
|
|
326
394
|
verbatim intent. Drop off-intent keywords, `count: 0`, and redundant
|
|
327
395
|
`subsumed_by` duplicates.
|
|
@@ -448,8 +516,14 @@ then fold the answers into the groups and the validators' TOPIC/NOT lines.
|
|
|
448
516
|
run ≥3 rounds, stop when fitness stops improving (sane cap ~6 rounds), note
|
|
449
517
|
that you ran autonomously.
|
|
450
518
|
|
|
519
|
+
*On the platform:* always run autonomously, measuring with `kw_probe` (mode
|
|
520
|
+
`sqs`) and re-validating with `kw_validate`.
|
|
521
|
+
|
|
451
522
|
### Stage 5 — Materialize the chosen deliverables
|
|
452
523
|
|
|
524
|
+
*On the platform:* skip this stage. The platform builds the deliverables from
|
|
525
|
+
your final groups.
|
|
526
|
+
|
|
453
527
|
Everything here takes the final filter verbatim via `--group`, so what you
|
|
454
528
|
deliver is exactly what the filter selects.
|
|
455
529
|
|
|
@@ -541,6 +615,11 @@ on-topic uploads. Tell the user when one channel dominates and offer
|
|
|
541
615
|
|
|
542
616
|
### Stage 6 — Deliver (`build_report.py`)
|
|
543
617
|
|
|
618
|
+
*On the platform:* don't run `build_report.py` and don't offer to save. Call
|
|
619
|
+
`submit_result` with the groups in the same shape (`text`, plus
|
|
620
|
+
`content_fields` / `exclude` where needed); the platform prunes them,
|
|
621
|
+
translates them into the report grammar and builds the link.
|
|
622
|
+
|
|
544
623
|
Hand the validated groups to the builder — start from `select_keywords.py
|
|
545
624
|
--apply`'s `groups`, adding per-group `content_fields` / `exclude` as needed:
|
|
546
625
|
|
|
@@ -39,7 +39,15 @@ Input (stdin JSON):
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
Output (stdout): a single JSON object — see OUTPUT CONTRACT at the bottom.
|
|
42
|
+
|
|
43
|
+
Library use: the file is import-safe (standard library only, nothing runs at import),
|
|
44
|
+
so another Python program can load it by path and call `build(spec, *, app_url=...)`.
|
|
45
|
+
`spec` is the parsed input object above; the return value is the object the command
|
|
46
|
+
prints. `build` leaves `spec` untouched and returns fresh structures. A bad spec raises
|
|
47
|
+
`ReportSpecError` (a `ValueError`). Library functions never print, read stdin or exit.
|
|
42
48
|
"""
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
43
51
|
import argparse
|
|
44
52
|
import json
|
|
45
53
|
import re
|
|
@@ -116,6 +124,10 @@ DEFAULT_FIELDS = {
|
|
|
116
124
|
}
|
|
117
125
|
|
|
118
126
|
|
|
127
|
+
class ReportSpecError(ValueError):
|
|
128
|
+
"""The spec handed to `build` is unusable."""
|
|
129
|
+
|
|
130
|
+
|
|
119
131
|
def tokens(text):
|
|
120
132
|
return _TOKEN_RE.findall(text.lower())
|
|
121
133
|
|
|
@@ -389,33 +401,23 @@ def build_report_config(filter_set, report_type_int, title, description):
|
|
|
389
401
|
}
|
|
390
402
|
|
|
391
403
|
|
|
392
|
-
def
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
args = ap.parse_args()
|
|
398
|
-
|
|
399
|
-
if sys.stdin.isatty():
|
|
400
|
-
sys.exit("pipe a JSON spec on stdin (see module docstring)")
|
|
401
|
-
try:
|
|
402
|
-
spec = json.loads(sys.stdin.read())
|
|
403
|
-
except json.JSONDecodeError as exc:
|
|
404
|
-
sys.exit(f"invalid JSON on stdin: {exc}")
|
|
405
|
-
|
|
406
|
-
operator = (spec.get("operator") or "OR").upper()
|
|
404
|
+
def build(spec, *, app_url=DEFAULT_APP_URL):
|
|
405
|
+
"""Build the deliverable (filter set, report link, report config) from a spec; raises ReportSpecError."""
|
|
406
|
+
if not isinstance(spec, dict):
|
|
407
|
+
raise ReportSpecError("spec must be a JSON object")
|
|
408
|
+
operator = str(spec.get("operator") or "OR").upper()
|
|
407
409
|
if operator not in ("AND", "OR"):
|
|
408
|
-
|
|
409
|
-
report_type = (spec.get("report_type") or "channels").lower()
|
|
410
|
+
raise ReportSpecError("operator must be AND or OR")
|
|
411
|
+
report_type = str(spec.get("report_type") or "channels").lower()
|
|
410
412
|
if report_type not in REPORT_TYPES:
|
|
411
|
-
|
|
413
|
+
raise ReportSpecError(f"report_type must be one of {sorted(REPORT_TYPES)}")
|
|
412
414
|
slug, report_type_int = REPORT_TYPES[report_type]
|
|
413
415
|
default_fields = spec.get("default_content_fields") or DEFAULT_FIELDS[report_type]
|
|
414
416
|
|
|
415
417
|
groups = spec.get("groups") or []
|
|
416
|
-
groups = [g for g in groups if isinstance(g, dict) and str(g.get("text", "")).strip()]
|
|
418
|
+
groups = [dict(g) for g in groups if isinstance(g, dict) and str(g.get("text", "")).strip()]
|
|
417
419
|
if not groups:
|
|
418
|
-
|
|
420
|
+
raise ReportSpecError("provide at least one group with non-empty 'text'")
|
|
419
421
|
for g in groups:
|
|
420
422
|
g["text"] = str(g["text"]).strip()
|
|
421
423
|
|
|
@@ -426,7 +428,7 @@ def main():
|
|
|
426
428
|
used_fields.update(g.get("content_fields") or [])
|
|
427
429
|
unknown = sorted(f for f in used_fields if f not in VALID_CONTENT_FIELDS)
|
|
428
430
|
if unknown:
|
|
429
|
-
|
|
431
|
+
raise ReportSpecError(
|
|
430
432
|
f"unknown content_fields {unknown}; use recognized content field names "
|
|
431
433
|
f"(e.g. title, summary, transcript, channel_description, "
|
|
432
434
|
f"channel_topic_description) — not raw ES paths. Valid: {sorted(VALID_CONTENT_FIELDS)}"
|
|
@@ -443,18 +445,18 @@ def main():
|
|
|
443
445
|
try:
|
|
444
446
|
app_text = sqs_to_app_syntax(g["text"])
|
|
445
447
|
except ValueError as exc:
|
|
446
|
-
|
|
448
|
+
raise ReportSpecError(f"group {g['text']!r}: {exc}")
|
|
447
449
|
if app_text != g["text"]:
|
|
448
450
|
translated.append({"from": g["text"], "to": app_text})
|
|
449
451
|
g["text"] = app_text
|
|
450
452
|
|
|
451
453
|
filter_set = build_filter_set(kept, operator, default_fields)
|
|
452
|
-
report_link = build_inline_link(kept, operator, slug, default_fields,
|
|
454
|
+
report_link = build_inline_link(kept, operator, slug, default_fields, app_url)
|
|
453
455
|
title = spec.get("title") or f"{report_type.title()} — keyword search"
|
|
454
456
|
description = spec.get("description") or ""
|
|
455
457
|
report_config = build_report_config(filter_set, report_type_int, title, description)
|
|
456
458
|
|
|
457
|
-
|
|
459
|
+
return {
|
|
458
460
|
"report_type": report_type,
|
|
459
461
|
"operator": operator,
|
|
460
462
|
"expression": render_expression(kept, operator),
|
|
@@ -463,7 +465,28 @@ def main():
|
|
|
463
465
|
"report_config": report_config,
|
|
464
466
|
"pruned": pruned,
|
|
465
467
|
"translated": translated,
|
|
466
|
-
}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def main():
|
|
472
|
+
ap = argparse.ArgumentParser(
|
|
473
|
+
description="Build the keyword-research deliverable (filter set + report link) from validated groups."
|
|
474
|
+
)
|
|
475
|
+
ap.add_argument("--app-url", default=DEFAULT_APP_URL, help=f"App base URL (default {DEFAULT_APP_URL})")
|
|
476
|
+
args = ap.parse_args()
|
|
477
|
+
|
|
478
|
+
if sys.stdin.isatty():
|
|
479
|
+
sys.exit("pipe a JSON spec on stdin (see module docstring)")
|
|
480
|
+
try:
|
|
481
|
+
spec = json.loads(sys.stdin.read())
|
|
482
|
+
except json.JSONDecodeError as exc:
|
|
483
|
+
sys.exit(f"invalid JSON on stdin: {exc}")
|
|
484
|
+
|
|
485
|
+
try:
|
|
486
|
+
out = build(spec, app_url=args.app_url)
|
|
487
|
+
except ReportSpecError as exc:
|
|
488
|
+
sys.exit(str(exc))
|
|
489
|
+
print(json.dumps(out, ensure_ascii=False, indent=2))
|
|
467
490
|
|
|
468
491
|
|
|
469
492
|
# OUTPUT CONTRACT (stdout, single JSON object):
|
|
@@ -33,8 +33,19 @@ Usage:
|
|
|
33
33
|
python3 expand_entities.py --existing "cannes lions" "advertising awards" < resolver.json > expanded.json
|
|
34
34
|
# pipe the candidates straight into probe.py:
|
|
35
35
|
python3 expand_entities.py --probe-batch < resolver.json | python3 probe.py --samples 5
|
|
36
|
+
|
|
37
|
+
Library use: the file is import-safe (standard library only, nothing runs at import),
|
|
38
|
+
so another Python program can load it by path and call `expand(resolver_reply, *,
|
|
39
|
+
existing=None)` (what the default mode prints) or `probe_batch(resolver_reply, *,
|
|
40
|
+
existing=None)` (what `--probe-batch` prints). `resolver_reply` is the parsed resolver
|
|
41
|
+
JSON; `existing` lists the candidates already brainstormed. Both return fresh structures
|
|
42
|
+
that share nothing with the input. Bad input raises `ExpandInputError` (a `ValueError`).
|
|
43
|
+
Library functions never print, read stdin or exit.
|
|
36
44
|
"""
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
37
47
|
import argparse
|
|
48
|
+
import copy
|
|
38
49
|
import json
|
|
39
50
|
import re
|
|
40
51
|
import sys
|
|
@@ -80,6 +91,10 @@ def solid(text):
|
|
|
80
91
|
return _NONWORD_RE.sub("", text)
|
|
81
92
|
|
|
82
93
|
|
|
94
|
+
class ExpandInputError(ValueError):
|
|
95
|
+
"""The resolver reply or options handed to the expander are unusable."""
|
|
96
|
+
|
|
97
|
+
|
|
83
98
|
def _norm(text):
|
|
84
99
|
"""Case/space-insensitive key for dedupe."""
|
|
85
100
|
return re.sub(r"\s+", " ", text.strip().lower())
|
|
@@ -156,8 +171,25 @@ def _collect_names(resolver, max_insider):
|
|
|
156
171
|
return fams
|
|
157
172
|
|
|
158
173
|
|
|
159
|
-
def
|
|
160
|
-
"""
|
|
174
|
+
def check_resolver(resolver_reply):
|
|
175
|
+
"""Raise ExpandInputError unless the resolver reply is a JSON object."""
|
|
176
|
+
if not isinstance(resolver_reply, dict):
|
|
177
|
+
raise ExpandInputError("resolver JSON must be an object")
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def expand(resolver_reply, *, existing=None, max_insider=MAX_INSIDER_DEFAULT):
|
|
181
|
+
"""Build probe candidates + family detail from a resolver JSON object (what the default mode prints).
|
|
182
|
+
|
|
183
|
+
`existing` lists candidates already brainstormed; matching forms are dropped.
|
|
184
|
+
`max_insider` caps the insider terms folded in. Raises ExpandInputError.
|
|
185
|
+
"""
|
|
186
|
+
check_resolver(resolver_reply)
|
|
187
|
+
resolver = resolver_reply
|
|
188
|
+
existing = [] if existing is None else existing
|
|
189
|
+
if not isinstance(existing, (list, tuple)) or not all(isinstance(x, str) for x in existing):
|
|
190
|
+
raise ExpandInputError("existing must be a list of strings")
|
|
191
|
+
if max_insider < 0:
|
|
192
|
+
raise ExpandInputError("--max-insider must be >= 0")
|
|
161
193
|
seen = {_norm(x) for x in existing} # forms already covered → skip
|
|
162
194
|
deduped = []
|
|
163
195
|
|
|
@@ -211,7 +243,7 @@ def expand(resolver, existing, max_insider):
|
|
|
211
243
|
)
|
|
212
244
|
collisions = [c for c in (resolver.get("collisions", []) or []) if isinstance(c, dict)]
|
|
213
245
|
|
|
214
|
-
|
|
246
|
+
result = {
|
|
215
247
|
"probe_candidates": probe_candidates,
|
|
216
248
|
"families": families,
|
|
217
249
|
"aliases": aliases,
|
|
@@ -228,6 +260,12 @@ def expand(resolver, existing, max_insider):
|
|
|
228
260
|
"deduped": len(deduped),
|
|
229
261
|
},
|
|
230
262
|
}
|
|
263
|
+
return copy.deepcopy(result) # the caller's input is never shared with the output
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def probe_batch(resolver_reply, *, existing=None, max_insider=MAX_INSIDER_DEFAULT):
|
|
267
|
+
"""Just the probe_candidates array (what --probe-batch prints), ready for probe.py."""
|
|
268
|
+
return expand(resolver_reply, existing=existing, max_insider=max_insider)["probe_candidates"]
|
|
231
269
|
|
|
232
270
|
|
|
233
271
|
def load_resolver():
|
|
@@ -240,8 +278,10 @@ def load_resolver():
|
|
|
240
278
|
data = json.loads(raw)
|
|
241
279
|
except json.JSONDecodeError as exc:
|
|
242
280
|
sys.exit(f"invalid resolver JSON on stdin: {exc}")
|
|
243
|
-
|
|
244
|
-
|
|
281
|
+
try:
|
|
282
|
+
check_resolver(data)
|
|
283
|
+
except ExpandInputError as exc:
|
|
284
|
+
sys.exit(str(exc))
|
|
245
285
|
return data
|
|
246
286
|
|
|
247
287
|
|
|
@@ -259,7 +299,11 @@ def main():
|
|
|
259
299
|
if args.max_insider < 0:
|
|
260
300
|
sys.exit("--max-insider must be >= 0")
|
|
261
301
|
|
|
262
|
-
|
|
302
|
+
resolver = load_resolver()
|
|
303
|
+
try:
|
|
304
|
+
out = expand(resolver, existing=args.existing, max_insider=args.max_insider)
|
|
305
|
+
except ExpandInputError as exc:
|
|
306
|
+
sys.exit(str(exc))
|
|
263
307
|
if args.probe_batch:
|
|
264
308
|
print(json.dumps(out["probe_candidates"], ensure_ascii=False))
|
|
265
309
|
else:
|
|
@@ -28,13 +28,24 @@ Usage:
|
|
|
28
28
|
echo '[{"label":"shop+amazon","sqs":"\"tiktok shop\" +amazon"}]' | probe.py --mode sqs
|
|
29
29
|
|
|
30
30
|
Output (stdout): a single JSON object — see OUTPUT CONTRACT at the bottom.
|
|
31
|
+
|
|
32
|
+
Library use: the file is import-safe (standard library only, nothing runs at import),
|
|
33
|
+
so another Python program can load it by path and call
|
|
34
|
+
`run_probe(candidates, *, run_es, fields, ...)`. It returns the same object the command
|
|
35
|
+
prints. `run_es(body) -> dict` is the caller's query function (an Elasticsearch body in,
|
|
36
|
+
the parsed response out); a candidate whose query raises is recorded under `failed`.
|
|
37
|
+
`fields` is required — the command's defaults live only in `main()`. Bad input raises
|
|
38
|
+
`ProbeInputError` (a `ValueError`). Library functions never print, read stdin or exit.
|
|
31
39
|
"""
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
32
42
|
import argparse
|
|
33
43
|
import datetime
|
|
34
44
|
import json
|
|
35
45
|
import re
|
|
36
46
|
import subprocess
|
|
37
47
|
import sys
|
|
48
|
+
from collections.abc import Callable
|
|
38
49
|
|
|
39
50
|
# Article-level (doc_type:article) text fields vs channel-level (doc_type:channel).
|
|
40
51
|
# `summary` holds the video's creator-written description (links, hashtags, promo)
|
|
@@ -140,12 +151,12 @@ def build_body(candidate, *, fields, level, samples, source_paths, since, until,
|
|
|
140
151
|
match = {
|
|
141
152
|
"simple_query_string": {
|
|
142
153
|
"query": candidate["value"],
|
|
143
|
-
"fields": fields,
|
|
154
|
+
"fields": list(fields),
|
|
144
155
|
"default_operator": "and",
|
|
145
156
|
}
|
|
146
157
|
}
|
|
147
158
|
else:
|
|
148
|
-
match = {"multi_match": {"query": candidate["value"], "type": "phrase", "fields": fields}}
|
|
159
|
+
match = {"multi_match": {"query": candidate["value"], "type": "phrase", "fields": list(fields)}}
|
|
149
160
|
|
|
150
161
|
# Always scope to YouTube uploads (format 4). At topic level also default to
|
|
151
162
|
# longform videos (content_type is article-only; channel docs have none).
|
|
@@ -180,7 +191,7 @@ def build_body(candidate, *, fields, level, samples, source_paths, since, until,
|
|
|
180
191
|
return {
|
|
181
192
|
"size": samples,
|
|
182
193
|
"track_total_hits": True,
|
|
183
|
-
"_source": source_paths,
|
|
194
|
+
"_source": list(source_paths),
|
|
184
195
|
"collapse": {"field": collapse_field},
|
|
185
196
|
"aggs": aggs,
|
|
186
197
|
"query": {"bool": {"filter": filters, "must": [match]}},
|
|
@@ -191,6 +202,10 @@ class ProbeError(Exception):
|
|
|
191
202
|
"""One candidate's ES probe failed (timeout, non-zero exit, bad JSON)."""
|
|
192
203
|
|
|
193
204
|
|
|
205
|
+
class ProbeInputError(ValueError):
|
|
206
|
+
"""The candidates or options handed to the probe are unusable."""
|
|
207
|
+
|
|
208
|
+
|
|
194
209
|
def run_es(body):
|
|
195
210
|
"""Run `tl db es - --json` with `body` on stdin; return parsed response.
|
|
196
211
|
|
|
@@ -324,7 +339,7 @@ def collect_candidates(argv_words):
|
|
|
324
339
|
return [ln.strip() for ln in raw.splitlines() if ln.strip()]
|
|
325
340
|
if isinstance(parsed, list):
|
|
326
341
|
return parsed
|
|
327
|
-
|
|
342
|
+
raise ProbeInputError("stdin JSON must be a list")
|
|
328
343
|
|
|
329
344
|
|
|
330
345
|
def normalize(raw_items, default_mode):
|
|
@@ -341,7 +356,7 @@ def normalize(raw_items, default_mode):
|
|
|
341
356
|
elif "keyword" in item:
|
|
342
357
|
value, mode = item["keyword"], item.get("mode", default_mode)
|
|
343
358
|
else:
|
|
344
|
-
|
|
359
|
+
raise ProbeInputError(f"candidate object missing value/sqs/phrase/keyword: {item}")
|
|
345
360
|
value = str(value).strip()
|
|
346
361
|
label = str(item.get("label", value)).strip() or value
|
|
347
362
|
else:
|
|
@@ -416,90 +431,76 @@ def annotate_recency(row, data, *, level, min_recent_channels, min_share):
|
|
|
416
431
|
row["thin"] = below_floor and not stale
|
|
417
432
|
|
|
418
433
|
|
|
419
|
-
def
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
help="How to interpret plain candidates: phrase match (default) "
|
|
432
|
-
"or simple_query_string (Boolean: + | - \"phrase\" () field^boost).")
|
|
433
|
-
ap.add_argument("--fields", default=None,
|
|
434
|
-
help="Comma-separated ES fields to search (defaults by --level).")
|
|
435
|
-
ap.add_argument("--samples", type=int, default=DEFAULT_SAMPLES,
|
|
436
|
-
help=f"Sample docs returned per candidate (default {DEFAULT_SAMPLES}, max {MAX_SAMPLES}; 0 = count only).")
|
|
437
|
-
ap.add_argument("--content-type", choices=list(CONTENT_TYPES), default="longform",
|
|
438
|
-
help="Topic-level video content type filter (default longform). "
|
|
439
|
-
"'all' drops the filter (include shorts + live). Ignored at "
|
|
440
|
-
"channel level (channel docs have no content type). "
|
|
441
|
-
"YouTube-only (format 4) is always enforced.")
|
|
442
|
-
ap.add_argument("--since", help="publication_date >= YYYY-MM-DD (topic level only)")
|
|
443
|
-
ap.add_argument("--until", help="publication_date <= YYYY-MM-DD (topic level only)")
|
|
444
|
-
# Recency validation: headline counts stay ALL-TIME; this layers an extra
|
|
445
|
-
# "is the content still active?" signal on the SAME query (no extra round-trip).
|
|
446
|
-
ap.add_argument("--no-recency", action="store_true",
|
|
447
|
-
help="Skip the recency/active-content check (pure all-time counts).")
|
|
448
|
-
ap.add_argument("--recency-months", type=int, default=12,
|
|
449
|
-
help="Topic-level recency window in months (default 12). "
|
|
450
|
-
"Channel level uses posts_per_90_days>0 (channel docs have no date).")
|
|
451
|
-
ap.add_argument("--min-recent-channels", type=int, default=5,
|
|
452
|
-
help="STALE floor: fewer than this many recent/active distinct channels (default 5).")
|
|
453
|
-
ap.add_argument("--min-recent-share", type=float, default=0.10,
|
|
454
|
-
help="Topic STALE share: recent_channels/channels below this AND below the floor → stale (default 0.10).")
|
|
455
|
-
ap.add_argument("--min-active-share", type=float, default=0.33,
|
|
456
|
-
help="Channel STALE share: active_channels/channels below this AND below the floor → stale (default 0.33).")
|
|
457
|
-
args = ap.parse_args()
|
|
458
|
-
|
|
459
|
-
if args.recency_months < 1:
|
|
460
|
-
sys.exit("--recency-months must be >= 1")
|
|
434
|
+
def check_options(*, fields, level, mode, content_type, operator, since, until, recency_months):
|
|
435
|
+
"""Raise ProbeInputError for an unusable option combination (first problem wins)."""
|
|
436
|
+
if level not in COLLAPSE_FIELD:
|
|
437
|
+
raise ProbeInputError(f"level must be one of {sorted(COLLAPSE_FIELD)}, got {level!r}")
|
|
438
|
+
if mode not in ("phrase", "sqs"):
|
|
439
|
+
raise ProbeInputError(f"mode must be phrase or sqs, got {mode!r}")
|
|
440
|
+
if content_type not in CONTENT_TYPES:
|
|
441
|
+
raise ProbeInputError(f"content_type must be one of {list(CONTENT_TYPES)}, got {content_type!r}")
|
|
442
|
+
if operator not in ("AND", "OR"):
|
|
443
|
+
raise ProbeInputError(f"operator must be AND or OR, got {operator!r}")
|
|
444
|
+
if recency_months < 1:
|
|
445
|
+
raise ProbeInputError("--recency-months must be >= 1")
|
|
461
446
|
|
|
462
447
|
# Date windowing only makes sense for articles — channel docs have no
|
|
463
448
|
# publication_date, so a range filter there silently zeroes every count.
|
|
464
|
-
if (
|
|
465
|
-
|
|
466
|
-
for flag, val in (("--since",
|
|
449
|
+
if (since or until) and level != "topic":
|
|
450
|
+
raise ProbeInputError("--since/--until apply only at --level topic (channel docs have no publication date)")
|
|
451
|
+
for flag, val in (("--since", since), ("--until", until)):
|
|
467
452
|
if val and not valid_date(val):
|
|
468
|
-
|
|
453
|
+
raise ProbeInputError(f"{flag} must be YYYY-MM-DD, got {val!r}")
|
|
469
454
|
|
|
470
|
-
|
|
471
|
-
|
|
455
|
+
if not isinstance(fields, (list, tuple)):
|
|
456
|
+
raise ProbeInputError("fields must be a list of ES field paths")
|
|
472
457
|
if not fields:
|
|
473
|
-
|
|
474
|
-
bad = [f for f in fields if not _FIELD_RE.match(f)]
|
|
458
|
+
raise ProbeInputError("--fields must list at least one ES field")
|
|
459
|
+
bad = [f for f in fields if not (isinstance(f, str) and _FIELD_RE.match(f))]
|
|
475
460
|
if bad:
|
|
476
|
-
|
|
461
|
+
raise ProbeInputError(f"invalid --fields entries {bad}; use ES field paths, optionally boosted (e.g. title^3)")
|
|
477
462
|
|
|
478
|
-
|
|
463
|
+
|
|
464
|
+
def run_probe(candidates: list, *, run_es: Callable[[dict], dict], fields: list[str], level: str = "topic",
|
|
465
|
+
mode: str = "phrase", samples: int = 5, content_type: str = "longform", since: str | None = None,
|
|
466
|
+
until: str | None = None, operator: str = "OR", recency: bool = True, recency_months: int = 12,
|
|
467
|
+
min_recent_channels: int = 5, min_recent_share: float = 0.10, min_active_share: float = 0.33) -> dict:
|
|
468
|
+
"""Probe every candidate through `run_es` and return the object `main()` prints.
|
|
469
|
+
|
|
470
|
+
`candidates` are strings or dicts (what `normalize` accepts). `run_es(body)` takes an
|
|
471
|
+
Elasticsearch body and returns the parsed response; if it raises, that candidate is
|
|
472
|
+
recorded under `failed` and the rest continue. Raises ProbeInputError on bad input.
|
|
473
|
+
"""
|
|
474
|
+
check_options(fields=fields, level=level, mode=mode, content_type=content_type, operator=operator,
|
|
475
|
+
since=since, until=until, recency_months=recency_months)
|
|
476
|
+
fields = list(fields)
|
|
477
|
+
if not isinstance(candidates, (list, tuple)):
|
|
478
|
+
raise ProbeInputError("candidates must be a list of strings or objects")
|
|
479
|
+
|
|
480
|
+
field_pairs = TOPIC_SAMPLE_FIELDS if level == "topic" else CHANNEL_SAMPLE_FIELDS
|
|
479
481
|
source_paths = [p for p, _ in field_pairs]
|
|
480
|
-
samples = min(MAX_SAMPLES, max(0,
|
|
482
|
+
samples = min(MAX_SAMPLES, max(0, samples))
|
|
481
483
|
|
|
482
|
-
candidates = normalize(
|
|
484
|
+
candidates = normalize(candidates, mode)
|
|
483
485
|
if not candidates:
|
|
484
|
-
|
|
486
|
+
raise ProbeInputError("provide at least one candidate (positional args or JSON array on stdin)")
|
|
485
487
|
|
|
486
|
-
recency = not args.no_recency
|
|
487
488
|
# Topic level needs a date cutoff; channel level uses a non-date sentinel so
|
|
488
489
|
# build_body emits the posts_per_90_days "active" agg instead.
|
|
489
490
|
recency_cutoff = None
|
|
490
491
|
if recency:
|
|
491
|
-
recency_cutoff = months_ago_iso(
|
|
492
|
+
recency_cutoff = months_ago_iso(recency_months) if level == "topic" else "active"
|
|
492
493
|
|
|
493
494
|
results, failed = [], []
|
|
494
495
|
for cand in candidates:
|
|
495
|
-
body = build_body(cand, fields=fields, level=
|
|
496
|
-
source_paths=source_paths, since=
|
|
497
|
-
recency_cutoff=recency_cutoff, content_type=
|
|
496
|
+
body = build_body(cand, fields=fields, level=level, samples=samples,
|
|
497
|
+
source_paths=source_paths, since=since, until=until,
|
|
498
|
+
recency_cutoff=recency_cutoff, content_type=content_type)
|
|
498
499
|
try:
|
|
499
500
|
data = run_es(body)
|
|
500
|
-
except
|
|
501
|
-
|
|
502
|
-
|
|
501
|
+
except Exception as exc:
|
|
502
|
+
failed.append({"keyword": cand["value"], "label": cand["label"],
|
|
503
|
+
"error": str(exc) or type(exc).__name__})
|
|
503
504
|
continue
|
|
504
505
|
documents = extract_total(data)
|
|
505
506
|
channels = extract_distinct(data)
|
|
@@ -510,7 +511,7 @@ def main():
|
|
|
510
511
|
row = {
|
|
511
512
|
"keyword": cand["value"],
|
|
512
513
|
"label": cand["label"],
|
|
513
|
-
"count": documents if
|
|
514
|
+
"count": documents if level == "topic" else channels,
|
|
514
515
|
"documents": documents,
|
|
515
516
|
"channels": channels,
|
|
516
517
|
"samples": extract_samples(data, field_pairs) if samples else [],
|
|
@@ -518,18 +519,17 @@ def main():
|
|
|
518
519
|
"subsumed_by": [],
|
|
519
520
|
}
|
|
520
521
|
if recency:
|
|
521
|
-
annotate_recency(row, data, level=
|
|
522
|
-
min_recent_channels=
|
|
523
|
-
min_share=(
|
|
524
|
-
else args.min_active_share))
|
|
522
|
+
annotate_recency(row, data, level=level,
|
|
523
|
+
min_recent_channels=min_recent_channels,
|
|
524
|
+
min_share=(min_recent_share if level == "topic" else min_active_share))
|
|
525
525
|
results.append(row)
|
|
526
526
|
|
|
527
|
-
mark_subsumed(results,
|
|
527
|
+
mark_subsumed(results, operator)
|
|
528
528
|
|
|
529
529
|
# `count` is 0 exactly when the document total is 0 (a non-empty match always
|
|
530
530
|
# reaches >=1 distinct channel), so it is the single no-match gate.
|
|
531
531
|
recency_keys = (("recent_documents", "recent_channels", "recent_channel_share")
|
|
532
|
-
if
|
|
532
|
+
if level == "topic" else ("active_channels", "active_channel_share"))
|
|
533
533
|
survivors, dropped = [], []
|
|
534
534
|
for r in results:
|
|
535
535
|
if r["count"] == 0:
|
|
@@ -547,11 +547,11 @@ def main():
|
|
|
547
547
|
|
|
548
548
|
# Echo the always-on scope so the caller (and user) sees what was filtered.
|
|
549
549
|
scope = {"format": "youtube"}
|
|
550
|
-
if
|
|
551
|
-
scope["content_type"] =
|
|
550
|
+
if level == "topic":
|
|
551
|
+
scope["content_type"] = content_type
|
|
552
552
|
out = {
|
|
553
|
-
"operator":
|
|
554
|
-
"level":
|
|
553
|
+
"operator": operator,
|
|
554
|
+
"level": level,
|
|
555
555
|
"fields": fields,
|
|
556
556
|
"scope": scope,
|
|
557
557
|
"keywords": survivors,
|
|
@@ -559,10 +559,69 @@ def main():
|
|
|
559
559
|
"failed": failed,
|
|
560
560
|
}
|
|
561
561
|
if recency:
|
|
562
|
-
out["recency"] = ({"months":
|
|
563
|
-
if
|
|
562
|
+
out["recency"] = ({"months": recency_months, "cutoff": recency_cutoff}
|
|
563
|
+
if level == "topic" else {"signal": "posts_per_90_days>0"})
|
|
564
|
+
return out
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def main():
|
|
568
|
+
ap = argparse.ArgumentParser(
|
|
569
|
+
description="Probe ES for keyword counts + validation samples; drop empty, annotate subsumed."
|
|
570
|
+
)
|
|
571
|
+
ap.add_argument("keywords", nargs="*", help="Candidates (or pipe a JSON array on stdin)")
|
|
572
|
+
ap.add_argument("--operator", choices=["AND", "OR"], default="OR",
|
|
573
|
+
help="How the caller will combine survivors downstream (default OR). "
|
|
574
|
+
"Echoed in output and drives subsumption annotation (OR only).")
|
|
575
|
+
ap.add_argument("--level", choices=["topic", "channel"], default="topic",
|
|
576
|
+
help="topic = videos/articles (doc_type:article); "
|
|
577
|
+
"channel = whole channels (doc_type:channel). Default topic.")
|
|
578
|
+
ap.add_argument("--mode", choices=["phrase", "sqs"], default="phrase",
|
|
579
|
+
help="How to interpret plain candidates: phrase match (default) "
|
|
580
|
+
"or simple_query_string (Boolean: + | - \"phrase\" () field^boost).")
|
|
581
|
+
ap.add_argument("--fields", default=None,
|
|
582
|
+
help="Comma-separated ES fields to search (defaults by --level).")
|
|
583
|
+
ap.add_argument("--samples", type=int, default=DEFAULT_SAMPLES,
|
|
584
|
+
help=f"Sample docs returned per candidate (default {DEFAULT_SAMPLES}, max {MAX_SAMPLES}; 0 = count only).")
|
|
585
|
+
ap.add_argument("--content-type", choices=list(CONTENT_TYPES), default="longform",
|
|
586
|
+
help="Topic-level video content type filter (default longform). "
|
|
587
|
+
"'all' drops the filter (include shorts + live). Ignored at "
|
|
588
|
+
"channel level (channel docs have no content type). "
|
|
589
|
+
"YouTube-only (format 4) is always enforced.")
|
|
590
|
+
ap.add_argument("--since", help="publication_date >= YYYY-MM-DD (topic level only)")
|
|
591
|
+
ap.add_argument("--until", help="publication_date <= YYYY-MM-DD (topic level only)")
|
|
592
|
+
# Recency validation: headline counts stay ALL-TIME; this layers an extra
|
|
593
|
+
# "is the content still active?" signal on the SAME query (no extra round-trip).
|
|
594
|
+
ap.add_argument("--no-recency", action="store_true",
|
|
595
|
+
help="Skip the recency/active-content check (pure all-time counts).")
|
|
596
|
+
ap.add_argument("--recency-months", type=int, default=12,
|
|
597
|
+
help="Topic-level recency window in months (default 12). "
|
|
598
|
+
"Channel level uses posts_per_90_days>0 (channel docs have no date).")
|
|
599
|
+
ap.add_argument("--min-recent-channels", type=int, default=5,
|
|
600
|
+
help="STALE floor: fewer than this many recent/active distinct channels (default 5).")
|
|
601
|
+
ap.add_argument("--min-recent-share", type=float, default=0.10,
|
|
602
|
+
help="Topic STALE share: recent_channels/channels below this AND below the floor → stale (default 0.10).")
|
|
603
|
+
ap.add_argument("--min-active-share", type=float, default=0.33,
|
|
604
|
+
help="Channel STALE share: active_channels/channels below this AND below the floor → stale (default 0.33).")
|
|
605
|
+
args = ap.parse_args()
|
|
606
|
+
|
|
607
|
+
fields = ([f.strip() for f in args.fields.split(",") if f.strip()] if args.fields
|
|
608
|
+
else (TOPIC_FIELDS if args.level == "topic" else CHANNEL_FIELDS))
|
|
609
|
+
options = dict(fields=fields, level=args.level, mode=args.mode, content_type=args.content_type,
|
|
610
|
+
operator=args.operator, since=args.since, until=args.until,
|
|
611
|
+
recency_months=args.recency_months)
|
|
612
|
+
try:
|
|
613
|
+
check_options(**options) # an unusable option outranks a malformed stdin
|
|
614
|
+
candidates = collect_candidates(args.keywords)
|
|
615
|
+
out = run_probe(candidates, run_es=run_es, samples=args.samples, recency=not args.no_recency,
|
|
616
|
+
min_recent_channels=args.min_recent_channels, min_recent_share=args.min_recent_share,
|
|
617
|
+
min_active_share=args.min_active_share, **options)
|
|
618
|
+
except ProbeInputError as exc:
|
|
619
|
+
sys.exit(str(exc))
|
|
620
|
+
|
|
621
|
+
for f in out["failed"]:
|
|
622
|
+
sys.stderr.write(f"probe failed for {f['keyword']!r}: {f['error']}\n")
|
|
564
623
|
print(json.dumps(out, ensure_ascii=False))
|
|
565
|
-
if failed and not
|
|
624
|
+
if out["failed"] and not out["keywords"] and not out["dropped"]:
|
|
566
625
|
sys.exit(1) # every candidate failed — the batch itself is broken
|
|
567
626
|
|
|
568
627
|
|
|
@@ -26,15 +26,56 @@ Usage:
|
|
|
26
26
|
python3 select_keywords.py --emit-batch < probe.json > batch.json
|
|
27
27
|
# ...send batch.json (+ "intent: ..." line) to keyword-relevance-validator → verdict.json...
|
|
28
28
|
python3 select_keywords.py --apply verdict.json < probe.json
|
|
29
|
+
|
|
30
|
+
Library use: the file is import-safe (standard library only, nothing runs at import),
|
|
31
|
+
so another Python program can load it by path and call `emit_batch(probe)` (the batch
|
|
32
|
+
list) and `apply_verdict_maps(probe, verdict_maps, *, allow_missing=False)` (the
|
|
33
|
+
`--apply` result). `probe` is the parsed `probe.py` output; `verdict_maps` is a list of
|
|
34
|
+
`{i: bool}` maps, one per validator pass. Bad input raises `SelectInputError` (a
|
|
35
|
+
`ValueError`); an incomplete verdict raises `IncompleteVerdictError` with `.missing`, the
|
|
36
|
+
sorted indices without a verdict. With `allow_missing=True` the same gap is reported
|
|
37
|
+
through `warnings.warn` (`MissingVerdictWarning`) instead. Library functions never print,
|
|
38
|
+
read stdin or exit. `apply_verdicts` is the command's own layer (prints, exits).
|
|
29
39
|
"""
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
30
42
|
import argparse
|
|
31
43
|
import json
|
|
32
44
|
import sys
|
|
45
|
+
import warnings
|
|
33
46
|
|
|
34
47
|
# Friendly content keys shown to the validator, by probe level.
|
|
35
48
|
BATCH_CONTENT_KEYS = {"topic": ["title", "summary"], "channel": ["name", "topic"]}
|
|
36
49
|
|
|
37
50
|
|
|
51
|
+
class SelectInputError(ValueError):
|
|
52
|
+
"""The probe output or verdicts handed to the selector are unusable."""
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class IncompleteVerdictError(SelectInputError):
|
|
56
|
+
"""Some batch samples have no verdict in any pass; `.missing` lists their indices."""
|
|
57
|
+
|
|
58
|
+
def __init__(self, message, missing):
|
|
59
|
+
super().__init__(message)
|
|
60
|
+
self.missing = list(missing)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class MissingVerdictWarning(UserWarning):
|
|
64
|
+
"""Verdicts left some batch samples unjudged (allow_missing=True)."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def check_probe(probe):
|
|
68
|
+
"""Raise SelectInputError unless `probe` has the shape probe.py prints."""
|
|
69
|
+
if not isinstance(probe, dict) or not isinstance(probe.get("keywords"), list):
|
|
70
|
+
raise SelectInputError("probe JSON must be an object with a 'keywords' list")
|
|
71
|
+
for kw in probe["keywords"]:
|
|
72
|
+
if not isinstance(kw, dict) or "keyword" not in kw:
|
|
73
|
+
raise SelectInputError("each probe keyword must be an object with a 'keyword'")
|
|
74
|
+
samples = kw.get("samples", [])
|
|
75
|
+
if not isinstance(samples, list) or not all(isinstance(smp, dict) for smp in samples):
|
|
76
|
+
raise SelectInputError("each probe keyword's 'samples' must be a list of objects")
|
|
77
|
+
|
|
78
|
+
|
|
38
79
|
def load_probe():
|
|
39
80
|
if sys.stdin.isatty():
|
|
40
81
|
sys.exit("pipe probe.py output JSON on stdin")
|
|
@@ -42,8 +83,10 @@ def load_probe():
|
|
|
42
83
|
probe = json.loads(sys.stdin.read())
|
|
43
84
|
except json.JSONDecodeError as exc:
|
|
44
85
|
sys.exit(f"invalid probe JSON on stdin: {exc}")
|
|
45
|
-
|
|
46
|
-
|
|
86
|
+
try:
|
|
87
|
+
check_probe(probe)
|
|
88
|
+
except SelectInputError as exc:
|
|
89
|
+
sys.exit(str(exc))
|
|
47
90
|
return probe
|
|
48
91
|
|
|
49
92
|
|
|
@@ -59,6 +102,8 @@ def build_index(probe):
|
|
|
59
102
|
|
|
60
103
|
|
|
61
104
|
def emit_batch(probe):
|
|
105
|
+
"""The flat validation batch the keyword-relevance-validator expects (what --emit-batch prints)."""
|
|
106
|
+
check_probe(probe)
|
|
62
107
|
level = probe.get("level", "topic")
|
|
63
108
|
keys = BATCH_CONTENT_KEYS.get(level, BATCH_CONTENT_KEYS["topic"])
|
|
64
109
|
batch = []
|
|
@@ -67,7 +112,7 @@ def emit_batch(probe):
|
|
|
67
112
|
for k in keys:
|
|
68
113
|
item[k] = sample.get(k)
|
|
69
114
|
batch.append(item)
|
|
70
|
-
|
|
115
|
+
return batch
|
|
71
116
|
|
|
72
117
|
|
|
73
118
|
def load_verdicts(paths):
|
|
@@ -106,30 +151,47 @@ def check_completeness(index, verdict_maps, allow_missing):
|
|
|
106
151
|
"""Every batch sample must have a verdict in at least one pass.
|
|
107
152
|
|
|
108
153
|
Cheap validators silently drop the tail of a long list, so a missing `i`
|
|
109
|
-
usually means truncation, not judgement. Default:
|
|
110
|
-
the missing indices so the caller re-sends just those samples to a
|
|
111
|
-
validator run.
|
|
112
|
-
samples then count as not-relevant, which biases toward
|
|
154
|
+
usually means truncation, not judgement. Default: raise IncompleteVerdictError
|
|
155
|
+
listing the missing indices so the caller re-sends just those samples to a
|
|
156
|
+
fresh validator run. With `allow_missing` the gap is returned as warning text
|
|
157
|
+
instead (missing samples then count as not-relevant, which biases toward
|
|
158
|
+
dropping). Returns None when every sample has a verdict.
|
|
113
159
|
"""
|
|
114
160
|
covered = set()
|
|
115
161
|
for vm in verdict_maps:
|
|
116
162
|
covered.update(vm)
|
|
117
163
|
missing = sorted(i for i, _, _ in index if i not in covered)
|
|
118
164
|
if not missing:
|
|
119
|
-
return
|
|
165
|
+
return None
|
|
120
166
|
msg = (f"{len(missing)} of {len(index)} batch samples have no verdict "
|
|
121
167
|
f"(missing i: {missing[:20]}{'…' if len(missing) > 20 else ''}) — "
|
|
122
168
|
f"the validator likely truncated; re-send those samples and merge.")
|
|
123
169
|
if allow_missing:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
sys.exit(msg + " (or pass --allow-missing to treat them as not relevant)")
|
|
170
|
+
return msg
|
|
171
|
+
raise IncompleteVerdictError(msg, missing)
|
|
127
172
|
|
|
128
173
|
|
|
129
|
-
def
|
|
174
|
+
def apply_verdict_maps(probe, verdict_maps, *, allow_missing=False):
|
|
175
|
+
"""Decide each keyword from the verdicts and return what --apply prints.
|
|
176
|
+
|
|
177
|
+
`verdict_maps` is a list of {i: bool} maps, one per validator pass. Raises
|
|
178
|
+
SelectInputError / IncompleteVerdictError; with `allow_missing` a gap is
|
|
179
|
+
announced through `warnings.warn` (MissingVerdictWarning) instead.
|
|
180
|
+
"""
|
|
181
|
+
out, warning = _apply(probe, verdict_maps, allow_missing)
|
|
182
|
+
if warning:
|
|
183
|
+
warnings.warn(warning, MissingVerdictWarning, stacklevel=2)
|
|
184
|
+
return out
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _apply(probe, verdict_maps, allow_missing):
|
|
188
|
+
"""The decision itself: (result, gap-warning text or None)."""
|
|
189
|
+
check_probe(probe)
|
|
190
|
+
if not isinstance(verdict_maps, (list, tuple)) or not all(isinstance(vm, dict) for vm in verdict_maps):
|
|
191
|
+
raise SelectInputError("verdict_maps must be a list of {i: relevant} maps")
|
|
130
192
|
index = build_index(probe)
|
|
131
193
|
level = probe.get("level", "topic")
|
|
132
|
-
check_completeness(index, verdict_maps, allow_missing)
|
|
194
|
+
warning = check_completeness(index, verdict_maps, allow_missing)
|
|
133
195
|
finals = {i: relevant_at(i, verdict_maps) for i, _, _ in index}
|
|
134
196
|
|
|
135
197
|
per_kw = {} # keyword -> {"relevant": n, "total": n, "samples":[(rel, sample)]}
|
|
@@ -188,6 +250,19 @@ def apply_verdicts(probe, verdict_maps, allow_missing=False):
|
|
|
188
250
|
}
|
|
189
251
|
if level != "channel":
|
|
190
252
|
out["candidate_videos"] = candidate_videos
|
|
253
|
+
return out, warning
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def apply_verdicts(probe, verdict_maps, allow_missing=False):
|
|
257
|
+
"""The command's --apply layer: print the result, exit on an incomplete verdict, warn on stderr."""
|
|
258
|
+
try:
|
|
259
|
+
out, warning = _apply(probe, verdict_maps, allow_missing)
|
|
260
|
+
except IncompleteVerdictError as exc:
|
|
261
|
+
sys.exit(f"{exc} (or pass --allow-missing to treat them as not relevant)")
|
|
262
|
+
except SelectInputError as exc:
|
|
263
|
+
sys.exit(str(exc))
|
|
264
|
+
if warning:
|
|
265
|
+
sys.stderr.write(f"warning: {warning}\n")
|
|
191
266
|
print(json.dumps(out, ensure_ascii=False, indent=2))
|
|
192
267
|
|
|
193
268
|
|
|
@@ -203,7 +278,7 @@ def main():
|
|
|
203
278
|
|
|
204
279
|
probe = load_probe()
|
|
205
280
|
if args.emit_batch:
|
|
206
|
-
emit_batch(probe)
|
|
281
|
+
print(json.dumps(emit_batch(probe), ensure_ascii=False))
|
|
207
282
|
else:
|
|
208
283
|
apply_verdicts(probe, load_verdicts(args.apply), allow_missing=args.allow_missing)
|
|
209
284
|
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
{"name": "Avg. Comments", "backend_code": "comments_avg", "sortability": "both", "description": "Average comments per YouTube video."},
|
|
42
42
|
{"name": "Median Evergreenness", "backend_code": "evergreenness_median", "sortability": "both", "description": "Median evergreenness for the channel."},
|
|
43
43
|
{"name": "Avg. Evergreenness", "backend_code": "evergreenness_avg", "sortability": "both", "description": "Average evergreenness for the channel."},
|
|
44
|
-
{"name": "Latest AdSpot Price", "backend_code": "latest_adspot_price", "sortability": "both", "description": "Price from the latest
|
|
44
|
+
{"name": "Latest AdSpot Price", "backend_code": "latest_adspot_price", "sortability": "both", "description": "Price from the latest preferred AdSpot."},
|
|
45
45
|
{"name": "Subscribers", "backend_code": "subscribers", "sortability": "both", "description": "Channel subscriber count."},
|
|
46
46
|
{"name": "Projected Views", "backend_code": "projected_views", "sortability": "both", "description": "Projected views for the channel's next video."},
|
|
47
47
|
{"name": "Trend", "backend_code": "trend", "sortability": "both", "description": "Growth trend score — higher means faster-growing channel."}
|
tl_cli/commands/channels.py
CHANGED
|
@@ -71,7 +71,7 @@ def show_cmd(
|
|
|
71
71
|
csv_output: bool = typer.Option(False, "--csv", help="CSV output (flattens adspots: one row per adspot)"),
|
|
72
72
|
toon_output: bool = typer.Option(False, "--toon", help="TOON output (token-efficient for LLMs)"),
|
|
73
73
|
) -> None:
|
|
74
|
-
"""Show channel detail by ID or name (includes
|
|
74
|
+
"""Show channel detail by ID or name (includes preferred adspots).
|
|
75
75
|
|
|
76
76
|
Accepts either a numeric channel ID or a partial name. Names that
|
|
77
77
|
match more than one active channel return a 400 with the candidate
|
|
File without changes
|
{thoughtleaders_cli-0.9.22.dist-info → thoughtleaders_cli-0.9.24.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{thoughtleaders_cli-0.9.22.dist-info → thoughtleaders_cli-0.9.24.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|