threadsense 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. threadsense-0.3.0/PKG-INFO +237 -0
  2. threadsense-0.3.0/README.md +204 -0
  3. {threadsense-0.2.0 → threadsense-0.3.0}/docs/artifacts.md +3 -0
  4. {threadsense-0.2.0 → threadsense-0.3.0}/docs/overview.md +6 -5
  5. {threadsense-0.2.0 → threadsense-0.3.0}/docs/system-design.md +5 -4
  6. {threadsense-0.2.0 → threadsense-0.3.0}/docs/usage.md +37 -2
  7. {threadsense-0.2.0 → threadsense-0.3.0}/pyproject.toml +1 -1
  8. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/__init__.py +1 -1
  9. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/cli.py +50 -2
  10. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/config.py +32 -1
  11. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/connectors/github_discussions.py +269 -30
  12. threadsense-0.3.0/src/threadsense/connectors/github_gist.py +504 -0
  13. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/connectors/registry.py +6 -1
  14. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/definitions/developer_tools.toml +5 -5
  15. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/errors.py +10 -0
  16. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/normalize.py +93 -0
  17. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_bootstrap.py +6 -1
  18. {threadsense-0.2.0 → threadsense-0.3.0}/tests/regression/expected_outputs/reddit_feedback_thread.json +3 -3
  19. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_analysis_pipeline.py +3 -0
  20. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_config.py +6 -1
  21. threadsense-0.3.0/tests/unit/test_github_gist_connector.py +355 -0
  22. {threadsense-0.2.0 → threadsense-0.3.0}/threadsense.toml +11 -2
  23. {threadsense-0.2.0 → threadsense-0.3.0}/uv.lock +1 -1
  24. threadsense-0.2.0/PKG-INFO +0 -249
  25. threadsense-0.2.0/README.md +0 -216
  26. {threadsense-0.2.0 → threadsense-0.3.0}/.approval_tests_temp/.gitignore +0 -0
  27. {threadsense-0.2.0 → threadsense-0.3.0}/.approval_tests_temp/approve_all.py +0 -0
  28. {threadsense-0.2.0 → threadsense-0.3.0}/.approval_tests_temp/remove_abandoned_files.py +0 -0
  29. {threadsense-0.2.0 → threadsense-0.3.0}/.gitignore +0 -0
  30. {threadsense-0.2.0 → threadsense-0.3.0}/LICENSE +0 -0
  31. {threadsense-0.2.0 → threadsense-0.3.0}/docs/batch-api-runtime.md +0 -0
  32. {threadsense-0.2.0 → threadsense-0.3.0}/docs/local-runtime-contract.md +0 -0
  33. {threadsense-0.2.0 → threadsense-0.3.0}/docs/output-modes.md +0 -0
  34. {threadsense-0.2.0 → threadsense-0.3.0}/docs/pitch.md +0 -0
  35. {threadsense-0.2.0 → threadsense-0.3.0}/docs/research-reddit.md +0 -0
  36. {threadsense-0.2.0 → threadsense-0.3.0}/scripts/validate.sh +0 -0
  37. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/__main__.py +0 -0
  38. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/action_signals.py +0 -0
  39. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/api_server.py +0 -0
  40. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/batching.py +0 -0
  41. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/cli_display.py +0 -0
  42. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/connectors/__init__.py +0 -0
  43. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/connectors/cache.py +0 -0
  44. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/connectors/hackernews.py +0 -0
  45. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/connectors/reddit.py +0 -0
  46. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/contracts/__init__.py +0 -0
  47. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/contracts/analysis_contract.py +0 -0
  48. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/contracts/defaults.py +0 -0
  49. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/__init__.py +0 -0
  50. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/definitions/financial_markets.toml +0 -0
  51. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/definitions/gaming.toml +0 -0
  52. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/definitions/hiring_careers.toml +0 -0
  53. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/definitions/product_feedback.toml +0 -0
  54. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/definitions/research_academic.toml +0 -0
  55. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/loader.py +0 -0
  56. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/domains/vocabulary.py +0 -0
  57. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/evaluation/__init__.py +0 -0
  58. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/evaluation/compare.py +0 -0
  59. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/evaluation/golden.py +0 -0
  60. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/evaluation/inference_quality.py +0 -0
  61. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/evaluation/metrics.py +0 -0
  62. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/inference/__init__.py +0 -0
  63. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/inference/contracts.py +0 -0
  64. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/inference/local_runtime.py +0 -0
  65. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/inference/prompts.py +0 -0
  66. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/inference/router.py +0 -0
  67. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/logging_config.py +0 -0
  68. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/models/__init__.py +0 -0
  69. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/models/analysis.py +0 -0
  70. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/models/canonical.py +0 -0
  71. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/models/corpus.py +0 -0
  72. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/models/report.py +0 -0
  73. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/models/results.py +0 -0
  74. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/observability.py +0 -0
  75. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/__init__.py +0 -0
  76. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/alignment.py +0 -0
  77. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/analyze.py +0 -0
  78. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/corpus.py +0 -0
  79. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/corpus_index.py +0 -0
  80. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/domain_detect.py +0 -0
  81. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/replay.py +0 -0
  82. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/storage.py +0 -0
  83. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/strategies/__init__.py +0 -0
  84. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/strategies/keyword_heuristic.py +0 -0
  85. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/tree.py +0 -0
  86. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/pipeline/versioning.py +0 -0
  87. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/preflight.py +0 -0
  88. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/py.typed +0 -0
  89. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/reporting/__init__.py +0 -0
  90. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/reporting/build.py +0 -0
  91. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/reporting/corpus_render.py +0 -0
  92. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/reporting/quality.py +0 -0
  93. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/reporting/render.py +0 -0
  94. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/schema_utils.py +0 -0
  95. {threadsense-0.2.0 → threadsense-0.3.0}/src/threadsense/workflows.py +0 -0
  96. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/README.md +0 -0
  97. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/analysis/canonical_feedback_thread.json +0 -0
  98. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/batch/reddit_manifest.json +0 -0
  99. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/hackernews/raw/story_thread.json +0 -0
  100. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/inference/analysis_summary_invalid_response.json +0 -0
  101. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/inference/analysis_summary_response.json +0 -0
  102. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/inference/chat_completion_request.json +0 -0
  103. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/inference/chat_completion_response.json +0 -0
  104. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/normalized/.gitkeep +0 -0
  105. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/reddit/raw/.gitkeep +0 -0
  106. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/reddit/raw/deleted_thread.json +0 -0
  107. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/reddit/raw/large_thread.json +0 -0
  108. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/reddit/raw/morechildren_response.json +0 -0
  109. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/reddit/raw/normal_thread.json +0 -0
  110. {threadsense-0.2.0 → threadsense-0.3.0}/tests/fixtures/reddit/raw/removed_thread.json +0 -0
  111. {threadsense-0.2.0 → threadsense-0.3.0}/tests/golden/README.md +0 -0
  112. {threadsense-0.2.0 → threadsense-0.3.0}/tests/golden/developer_tools/reddit_feedback_thread.json +0 -0
  113. {threadsense-0.2.0 → threadsense-0.3.0}/tests/golden/manifest.json +0 -0
  114. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_batch_api.py +0 -0
  115. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_canonical_persistence.py +0 -0
  116. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_full_pipeline.py +0 -0
  117. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_local_inference.py +0 -0
  118. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_preflight.py +0 -0
  119. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_reddit_fetch.py +0 -0
  120. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_reddit_research.py +0 -0
  121. {threadsense-0.2.0 → threadsense-0.3.0}/tests/integration/test_reporting.py +0 -0
  122. {threadsense-0.2.0 → threadsense-0.3.0}/tests/property/__init__.py +0 -0
  123. {threadsense-0.2.0 → threadsense-0.3.0}/tests/property/test_schema_properties.py +0 -0
  124. {threadsense-0.2.0 → threadsense-0.3.0}/tests/regression/test_analysis_replay.py +0 -0
  125. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_alignment.py +0 -0
  126. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_api_server.py +0 -0
  127. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_artifact_versioning.py +0 -0
  128. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_batching.py +0 -0
  129. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_canonical_models.py +0 -0
  130. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_cli.py +0 -0
  131. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_cli_display.py +0 -0
  132. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_corpus_analysis.py +0 -0
  133. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_corpus_index.py +0 -0
  134. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_corpus_model.py +0 -0
  135. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_domain_detect.py +0 -0
  136. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_errors.py +0 -0
  137. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_fetch_cache.py +0 -0
  138. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_github_discussions_connector.py +0 -0
  139. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_golden_evaluation.py +0 -0
  140. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_hackernews_connector.py +0 -0
  141. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_inference_quality.py +0 -0
  142. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_inference_router.py +0 -0
  143. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_local_runtime.py +0 -0
  144. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_minhash.py +0 -0
  145. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_normalize_pipeline.py +0 -0
  146. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_observability.py +0 -0
  147. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_reddit_connector.py +0 -0
  148. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_replay.py +0 -0
  149. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_report_rendering.py +0 -0
  150. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_source_registry.py +0 -0
  151. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_strategy_comparison.py +0 -0
  152. {threadsense-0.2.0 → threadsense-0.3.0}/tests/unit/test_tree.py +0 -0
@@ -0,0 +1,237 @@
1
+ Metadata-Version: 2.4
2
+ Name: threadsense
3
+ Version: 0.3.0
4
+ Summary: Evidence-first discussion intelligence tooling.
5
+ License: MIT License
6
+
7
+ Copyright (c) 2026 Tom Mathews
8
+
9
+ Permission is hereby granted, free of charge, to any person obtaining a copy
10
+ of this software and associated documentation files (the "Software"), to deal
11
+ in the Software without restriction, including without limitation the rights
12
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13
+ copies of the Software, and to permit persons to whom the Software is
14
+ furnished to do so, subject to the following conditions:
15
+
16
+ The above copyright notice and this permission notice shall be included in all
17
+ copies or substantial portions of the Software.
18
+
19
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
+ SOFTWARE.
26
+ License-File: LICENSE
27
+ Requires-Python: >=3.11
28
+ Requires-Dist: httpx>=0.28.1
29
+ Requires-Dist: pydantic-settings>=2.13.1
30
+ Requires-Dist: pydantic>=2.12.5
31
+ Requires-Dist: rich>=14.3.3
32
+ Description-Content-Type: text/markdown
33
+
34
+ # ThreadSense
35
+
36
+ **Discussion intelligence, not scraping.** ThreadSense is a reproducible pipeline that turns public discussion threads into structured, evidence-backed product and research intelligence.
37
+
38
+ ## Why ThreadSense
39
+
40
+ Scrapers give you raw payloads. AI summarizers give you prose. Neither gives you a defensible basis for decisions.
41
+
42
+ ThreadSense keeps the evidence chain intact at every stage:
43
+
44
+ 1. **Acquire** — source connectors fetch threads from Reddit, Hacker News, GitHub Discussions, and GitHub Gists
45
+ 2. **Normalize** — source-specific payloads are mapped into a canonical thread model with provenance metadata
46
+ 3. **Analyze** — deterministic extraction identifies issues, feature requests, themes, and sentiment — each linked to the comment that produced it
47
+ 4. **Synthesize** — optional local-model inference adds summaries on top of the deterministic evidence layer
48
+ 5. **Report** — structured outputs in Markdown, HTML, or JSON, with full traceability from finding back to source comment
49
+
50
+ Every stage produces a persisted, inspectable artifact. Rerun any stage independently. Diff results across runs. Audit exactly where a finding came from.
51
+
52
+ ## What This Enables
53
+
54
+ ### Single-thread analysis
55
+
56
+ Feed a discussion URL. Get structured findings — issues, requests, themes, severity — with every claim linked to the comment that produced it.
57
+
58
+ ```bash
59
+ uv run threadsense run reddit \
60
+ "https://www.reddit.com/r/ClaudeCode/comments/1ro0qbl/anyone_actually_built_a_second_brain_that_isnt/" \
61
+ --format markdown \
62
+ --with-summary \
63
+ --summary-required
64
+ ```
65
+
66
+ Works with GitHub Gists too — flat-comment discussions with code context:
67
+
68
+ ```bash
69
+ uv run threadsense run gist \
70
+ "https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f" \
71
+ --format markdown
72
+ ```
73
+
74
+ ### Cross-thread research
75
+
76
+ Search a topic across multiple subreddits. ThreadSense deterministically selects, ranks, and analyzes matching threads, then synthesizes a corpus-level report.
77
+
78
+ ```bash
79
+ uv run threadsense --output-format human research reddit \
80
+ --query "second brain OR agentic PKM" \
81
+ --subreddit ClaudeCode \
82
+ --subreddit LocalLLaMA \
83
+ --subreddit AI_Agents \
84
+ --limit 5 \
85
+ --per-subreddit-limit 3 \
86
+ --with-summary
87
+ ```
88
+
89
+ ### Domain-aware analysis
90
+
91
+ The analysis layer uses a contract system with domain-specific vocabularies (developer tools, product feedback, hiring, research, financial markets, gaming). Each domain defines its own theme keywords, issue markers, and severity calibration — so analysis adapts to context rather than applying one-size-fits-all heuristics.
92
+
93
+ ## Architecture
94
+
95
+ ```text
96
+ fetch → normalize → analyze → [optional inference] → report
97
+ ↓ ↓ ↓ ↓
98
+ raw artifact canonical analysis report artifact
99
+ artifact artifact
100
+ ```
101
+
102
+ - **Deterministic core** — parsing, normalization, scoring, and selection are reproducible across runs
103
+ - **Inference on top** — LLM synthesis is optional and layered over deterministic evidence, never a substitute for it
104
+ - **Stable artifacts** — each stage persists a separate JSON artifact with `schema_version` and SHA256 provenance
105
+ - **Fail fast** — invalid URLs, malformed payloads, and schema inconsistencies surface immediately
106
+
107
+ ## Sources and Discovery
108
+
109
+ | Capability | Reddit | Hacker News | GitHub Discussions | GitHub Gists |
110
+ | --------------- | :----: | :---------: | :----------------: | :----------: |
111
+ | Thread analysis | yes | yes | yes | yes |
112
+ | Topic research | yes | — | — | — |
113
+
114
+ ## Output Modes
115
+
116
+ | Mode | Purpose |
117
+ | ------- | ------------------------------------------------ |
118
+ | `json` | Machine-readable payloads for downstream tooling |
119
+ | `human` | Rich terminal panels and summaries for operators |
120
+ | `quiet` | Status-only output for scripts and CI |
121
+
122
+ ```bash
123
+ uv run threadsense --output-format human research reddit ...
124
+ ```
125
+
126
+ See [docs/output-modes.md](docs/output-modes.md) for details.
127
+
128
+ ## Who This Is For
129
+
130
+ - **Product teams** validating pain points and feature demand from community discussions
131
+ - **Founders** doing market and competitor research across technical communities
132
+ - **DevRel teams** tracking developer workflow friction and tooling sentiment
133
+ - **Researchers** studying technical communities with reproducible methodology
134
+
135
+ ## Quickstart
136
+
137
+ ```bash
138
+ # 1. Install
139
+ uv sync
140
+
141
+ # 2. Validate local setup
142
+ uv run threadsense preflight
143
+
144
+ # 3. Analyze a single thread
145
+ uv run threadsense run reddit \
146
+ "https://www.reddit.com/r/ClaudeCode/comments/1ro0qbl/anyone_actually_built_a_second_brain_that_isnt/"
147
+
148
+ # 4. Analyze a GitHub Gist
149
+ uv run threadsense run gist \
150
+ "https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f"
151
+
152
+ # 5. Research a topic across subreddits
153
+ uv run threadsense research reddit \
154
+ --query "second brain OR agentic PKM" \
155
+ --subreddit ClaudeCode \
156
+ --subreddit LocalLLaMA \
157
+ --subreddit AI_Agents
158
+ ```
159
+
160
+ ## CLI Commands
161
+
162
+ | Command | Purpose |
163
+ | ----------------- | --------------------------------------------------- |
164
+ | `run` | End-to-end single-thread pipeline |
165
+ | `research reddit` | Cross-subreddit topic research and corpus synthesis |
166
+ | `fetch` | Acquire raw thread data |
167
+ | `normalize` | Map raw data to canonical model |
168
+ | `analyze` | Deterministic evidence extraction |
169
+ | `infer` | LLM-assisted synthesis |
170
+ | `report` | Generate output reports |
171
+ | `corpus` | Build and analyze cross-thread corpora |
172
+ | `inspect` | Examine persisted artifacts |
173
+ | `batch run` | Process multiple threads |
174
+ | `preflight` | Validate local environment |
175
+ | `serve` | Local API server |
176
+
177
+ Full command reference: [docs/usage.md](docs/usage.md)
178
+
179
+ ## Artifact Storage
180
+
181
+ Every pipeline run produces inspectable artifacts under `.threadsense/`:
182
+
183
+ ```text
184
+ .threadsense/
185
+ ├── raw/<source>/ # Source payloads as fetched
186
+ ├── normalized/<source>/ # Canonical thread model
187
+ ├── analysis/<source>/ # Evidence-linked findings
188
+ ├── reports/<source>/ # Rendered reports
189
+ ├── corpora/<corpus-id>/ # Manifest, analysis, and report
190
+ └── batches/ # Batch run metadata
191
+ ```
192
+
193
+ Details: [docs/artifacts.md](docs/artifacts.md)
194
+
195
+ ## Local Runtime
196
+
197
+ ThreadSense runs without a local model for deterministic analysis. Summaries and synthesis require a local OpenAI-compatible endpoint (default: `http://127.0.0.1:8080/v1/chat/completions`).
198
+
199
+ Details: [docs/local-runtime-contract.md](docs/local-runtime-contract.md)
200
+
201
+ ## Documentation
202
+
203
+ | Document | Content |
204
+ | ----------------------------------------------------------- | ----------------------------------- |
205
+ | [usage.md](docs/usage.md) | Command reference |
206
+ | [research-reddit.md](docs/research-reddit.md) | Reddit topic research workflow |
207
+ | [output-modes.md](docs/output-modes.md) | JSON, human, and quiet output modes |
208
+ | [artifacts.md](docs/artifacts.md) | Artifact types and storage layout |
209
+ | [overview.md](docs/overview.md) | Product and workflow overview |
210
+ | [system-design.md](docs/system-design.md) | Architecture and system boundaries |
211
+ | [local-runtime-contract.md](docs/local-runtime-contract.md) | Local inference contract |
212
+ | [pitch.md](docs/pitch.md) | Product positioning |
213
+
214
+ ## Validation
215
+
216
+ ```bash
217
+ uv run ruff check
218
+ uv run ruff format --check .
219
+ uv run mypy --strict src tests
220
+ uv run pytest
221
+ ```
222
+
223
+ ## Current Limits
224
+
225
+ - Topic research is implemented for Reddit; other source discovery workflows are planned
226
+ - Reddit research queries support `OR`/`|` clause unions only (intentionally narrow for deterministic alignment)
227
+ - Corpus reports are Markdown only
228
+ - GitHub Gist comments are flat (no threading); analysis uses temporal/semantic clustering
229
+ - GitHub Gist connector does not retry on transient API errors (502/503)
230
+ - The local API is a trusted local surface, not a hardened public service
231
+
232
+ ## Direction
233
+
234
+ - Richer corpus presentation and operator workflows
235
+ - Discovery workflows beyond Reddit
236
+ - Evaluation and replay benchmarking
237
+ - Source-distribution and research-quality reporting
@@ -0,0 +1,204 @@
1
+ # ThreadSense
2
+
3
+ **Discussion intelligence, not scraping.** ThreadSense is a reproducible pipeline that turns public discussion threads into structured, evidence-backed product and research intelligence.
4
+
5
+ ## Why ThreadSense
6
+
7
+ Scrapers give you raw payloads. AI summarizers give you prose. Neither gives you a defensible basis for decisions.
8
+
9
+ ThreadSense keeps the evidence chain intact at every stage:
10
+
11
+ 1. **Acquire** — source connectors fetch threads from Reddit, Hacker News, GitHub Discussions, and GitHub Gists
12
+ 2. **Normalize** — source-specific payloads are mapped into a canonical thread model with provenance metadata
13
+ 3. **Analyze** — deterministic extraction identifies issues, feature requests, themes, and sentiment — each linked to the comment that produced it
14
+ 4. **Synthesize** — optional local-model inference adds summaries on top of the deterministic evidence layer
15
+ 5. **Report** — structured outputs in Markdown, HTML, or JSON, with full traceability from finding back to source comment
16
+
17
+ Every stage produces a persisted, inspectable artifact. Rerun any stage independently. Diff results across runs. Audit exactly where a finding came from.
18
+
19
+ ## What This Enables
20
+
21
+ ### Single-thread analysis
22
+
23
+ Feed a discussion URL. Get structured findings — issues, requests, themes, severity — with every claim linked to the comment that produced it.
24
+
25
+ ```bash
26
+ uv run threadsense run reddit \
27
+ "https://www.reddit.com/r/ClaudeCode/comments/1ro0qbl/anyone_actually_built_a_second_brain_that_isnt/" \
28
+ --format markdown \
29
+ --with-summary \
30
+ --summary-required
31
+ ```
32
+
33
+ Works with GitHub Gists too — flat-comment discussions with code context:
34
+
35
+ ```bash
36
+ uv run threadsense run gist \
37
+ "https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f" \
38
+ --format markdown
39
+ ```
40
+
41
+ ### Cross-thread research
42
+
43
+ Search a topic across multiple subreddits. ThreadSense deterministically selects, ranks, and analyzes matching threads, then synthesizes a corpus-level report.
44
+
45
+ ```bash
46
+ uv run threadsense --output-format human research reddit \
47
+ --query "second brain OR agentic PKM" \
48
+ --subreddit ClaudeCode \
49
+ --subreddit LocalLLaMA \
50
+ --subreddit AI_Agents \
51
+ --limit 5 \
52
+ --per-subreddit-limit 3 \
53
+ --with-summary
54
+ ```
55
+
56
+ ### Domain-aware analysis
57
+
58
+ The analysis layer uses a contract system with domain-specific vocabularies (developer tools, product feedback, hiring, research, financial markets, gaming). Each domain defines its own theme keywords, issue markers, and severity calibration — so analysis adapts to context rather than applying one-size-fits-all heuristics.
59
+
60
+ ## Architecture
61
+
62
+ ```text
63
+ fetch → normalize → analyze → [optional inference] → report
64
+ ↓ ↓ ↓ ↓
65
+ raw artifact canonical analysis report artifact
66
+ artifact artifact
67
+ ```
68
+
69
+ - **Deterministic core** — parsing, normalization, scoring, and selection are reproducible across runs
70
+ - **Inference on top** — LLM synthesis is optional and layered over deterministic evidence, never a substitute for it
71
+ - **Stable artifacts** — each stage persists a separate JSON artifact with `schema_version` and SHA256 provenance
72
+ - **Fail fast** — invalid URLs, malformed payloads, and schema inconsistencies surface immediately
73
+
74
+ ## Sources and Discovery
75
+
76
+ | Capability | Reddit | Hacker News | GitHub Discussions | GitHub Gists |
77
+ | --------------- | :----: | :---------: | :----------------: | :----------: |
78
+ | Thread analysis | yes | yes | yes | yes |
79
+ | Topic research | yes | — | — | — |
80
+
81
+ ## Output Modes
82
+
83
+ | Mode | Purpose |
84
+ | ------- | ------------------------------------------------ |
85
+ | `json` | Machine-readable payloads for downstream tooling |
86
+ | `human` | Rich terminal panels and summaries for operators |
87
+ | `quiet` | Status-only output for scripts and CI |
88
+
89
+ ```bash
90
+ uv run threadsense --output-format human research reddit ...
91
+ ```
92
+
93
+ See [docs/output-modes.md](docs/output-modes.md) for details.
94
+
95
+ ## Who This Is For
96
+
97
+ - **Product teams** validating pain points and feature demand from community discussions
98
+ - **Founders** doing market and competitor research across technical communities
99
+ - **DevRel teams** tracking developer workflow friction and tooling sentiment
100
+ - **Researchers** studying technical communities with reproducible methodology
101
+
102
+ ## Quickstart
103
+
104
+ ```bash
105
+ # 1. Install
106
+ uv sync
107
+
108
+ # 2. Validate local setup
109
+ uv run threadsense preflight
110
+
111
+ # 3. Analyze a single thread
112
+ uv run threadsense run reddit \
113
+ "https://www.reddit.com/r/ClaudeCode/comments/1ro0qbl/anyone_actually_built_a_second_brain_that_isnt/"
114
+
115
+ # 4. Analyze a GitHub Gist
116
+ uv run threadsense run gist \
117
+ "https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f"
118
+
119
+ # 5. Research a topic across subreddits
120
+ uv run threadsense research reddit \
121
+ --query "second brain OR agentic PKM" \
122
+ --subreddit ClaudeCode \
123
+ --subreddit LocalLLaMA \
124
+ --subreddit AI_Agents
125
+ ```
126
+
127
+ ## CLI Commands
128
+
129
+ | Command | Purpose |
130
+ | ----------------- | --------------------------------------------------- |
131
+ | `run` | End-to-end single-thread pipeline |
132
+ | `research reddit` | Cross-subreddit topic research and corpus synthesis |
133
+ | `fetch` | Acquire raw thread data |
134
+ | `normalize` | Map raw data to canonical model |
135
+ | `analyze` | Deterministic evidence extraction |
136
+ | `infer` | LLM-assisted synthesis |
137
+ | `report` | Generate output reports |
138
+ | `corpus` | Build and analyze cross-thread corpora |
139
+ | `inspect` | Examine persisted artifacts |
140
+ | `batch run` | Process multiple threads |
141
+ | `preflight` | Validate local environment |
142
+ | `serve` | Local API server |
143
+
144
+ Full command reference: [docs/usage.md](docs/usage.md)
145
+
146
+ ## Artifact Storage
147
+
148
+ Every pipeline run produces inspectable artifacts under `.threadsense/`:
149
+
150
+ ```text
151
+ .threadsense/
152
+ ├── raw/<source>/ # Source payloads as fetched
153
+ ├── normalized/<source>/ # Canonical thread model
154
+ ├── analysis/<source>/ # Evidence-linked findings
155
+ ├── reports/<source>/ # Rendered reports
156
+ ├── corpora/<corpus-id>/ # Manifest, analysis, and report
157
+ └── batches/ # Batch run metadata
158
+ ```
159
+
160
+ Details: [docs/artifacts.md](docs/artifacts.md)
161
+
162
+ ## Local Runtime
163
+
164
+ ThreadSense runs without a local model for deterministic analysis. Summaries and synthesis require a local OpenAI-compatible endpoint (default: `http://127.0.0.1:8080/v1/chat/completions`).
165
+
166
+ Details: [docs/local-runtime-contract.md](docs/local-runtime-contract.md)
167
+
168
+ ## Documentation
169
+
170
+ | Document | Content |
171
+ | ----------------------------------------------------------- | ----------------------------------- |
172
+ | [usage.md](docs/usage.md) | Command reference |
173
+ | [research-reddit.md](docs/research-reddit.md) | Reddit topic research workflow |
174
+ | [output-modes.md](docs/output-modes.md) | JSON, human, and quiet output modes |
175
+ | [artifacts.md](docs/artifacts.md) | Artifact types and storage layout |
176
+ | [overview.md](docs/overview.md) | Product and workflow overview |
177
+ | [system-design.md](docs/system-design.md) | Architecture and system boundaries |
178
+ | [local-runtime-contract.md](docs/local-runtime-contract.md) | Local inference contract |
179
+ | [pitch.md](docs/pitch.md) | Product positioning |
180
+
181
+ ## Validation
182
+
183
+ ```bash
184
+ uv run ruff check
185
+ uv run ruff format --check .
186
+ uv run mypy --strict src tests
187
+ uv run pytest
188
+ ```
189
+
190
+ ## Current Limits
191
+
192
+ - Topic research is implemented for Reddit; other source discovery workflows are planned
193
+ - Reddit research queries support `OR`/`|` clause unions only (intentionally narrow for deterministic alignment)
194
+ - Corpus reports are Markdown only
195
+ - GitHub Gist comments are flat (no threading); analysis uses temporal/semantic clustering
196
+ - GitHub Gist connector does not retry on transient API errors (502/503)
197
+ - The local API is a trusted local surface, not a hardened public service
198
+
199
+ ## Direction
200
+
201
+ - Richer corpus presentation and operator workflows
202
+ - Discovery workflows beyond Reddit
203
+ - Evaluation and replay benchmarking
204
+ - Source-distribution and research-quality reporting
@@ -25,6 +25,7 @@ Examples:
25
25
  - `.threadsense/raw/reddit/<thread-id>.json`
26
26
  - `.threadsense/raw/hackernews/<story-id>.json`
27
27
  - `.threadsense/raw/github_discussions/<thread-id>.json`
28
+ - `.threadsense/raw/github_gist/<gist-id>.json`
28
29
 
29
30
  Purpose:
30
31
 
@@ -41,6 +42,7 @@ Examples:
41
42
  - `.threadsense/normalized/reddit/<thread-id>.json`
42
43
  - `.threadsense/normalized/hackernews/<thread-id>.json`
43
44
  - `.threadsense/normalized/github_discussions/<thread-id>.json`
45
+ - `.threadsense/normalized/github_gist/<gist-id>.json`
44
46
 
45
47
  Purpose:
46
48
 
@@ -57,6 +59,7 @@ Examples:
57
59
  - `.threadsense/analysis/reddit/<thread-id>.json`
58
60
  - `.threadsense/analysis/hackernews/<thread-id>.json`
59
61
  - `.threadsense/analysis/github_discussions/<thread-id>.json`
62
+ - `.threadsense/analysis/github_gist/<gist-id>.json`
60
63
 
61
64
  Purpose:
62
65
 
@@ -6,7 +6,7 @@ ThreadSense is an evidence-first discussion intelligence system for turning publ
6
6
 
7
7
  The implemented system supports:
8
8
 
9
- - source connectors for Reddit, Hacker News, and GitHub Discussions
9
+ - source connectors for Reddit, Hacker News, GitHub Discussions, and GitHub Gists
10
10
  - canonical normalization into a shared thread model
11
11
  - deterministic analysis with evidence-linked findings
12
12
  - optional local-model synthesis for thread and corpus summaries
@@ -90,9 +90,10 @@ ThreadSense separates source-specific acquisition from canonical normalization.
90
90
 
91
91
  Current source support:
92
92
 
93
- - Reddit
94
- - Hacker News
95
- - GitHub Discussions
93
+ - Reddit — threaded comments via JSON API with "more" expansion
94
+ - Hacker News — recursive tree-based item fetching via Firebase API
95
+ - GitHub Discussions — GraphQL with cursor-based pagination for comments and replies
96
+ - GitHub Gists — REST API with paginated flat comments; authentication optional for public gists
96
97
 
97
98
  Each connector preserves enough source metadata for traceability, while normalization maps the result into a canonical `Thread` object with:
98
99
 
@@ -182,7 +183,7 @@ JSON is for machines. Human mode is for operators. Both should expose the same u
182
183
  ThreadSense is best described as:
183
184
 
184
185
  ```text
185
- multi-source thread analysis + corpus synthesis + subreddit topic research
186
+ multi-source discussion intelligence + corpus synthesis + subreddit topic research
186
187
  ```
187
188
 
188
189
  It is not just a scraper, and it is not just a summarizer. It is a reproducible pipeline from source thread to inspectable intelligence.
@@ -49,6 +49,7 @@ subreddit search -> candidate ranking -> selected thread fetch/analyze
49
49
  - `src/threadsense/connectors/reddit.py`
50
50
  - `src/threadsense/connectors/hackernews.py`
51
51
  - `src/threadsense/connectors/github_discussions.py`
52
+ - `src/threadsense/connectors/github_gist.py`
52
53
 
53
54
  Responsibilities:
54
55
 
@@ -58,11 +59,11 @@ Responsibilities:
58
59
  - preserve source-specific thread metadata
59
60
  - return raw artifacts for later normalization
60
61
 
61
- Special note for Reddit:
62
+ Special notes:
62
63
 
63
- - direct thread fetch is URL-based
64
- - topic research uses explicit subreddit search
65
- - exact time-window filtering is applied after Reddit retrieval
64
+ - Reddit: direct thread fetch is URL-based; topic research uses explicit subreddit search with exact time-window filtering after retrieval
65
+ - GitHub Discussions: GraphQL with cursor-based pagination for comments (100/page) and replies; requires authentication token
66
+ - GitHub Gists: REST API with paginated flat comments; authentication optional for public gists; comments have no threading (all depth=0)
66
67
 
67
68
  ### Canonical Model
68
69
 
@@ -43,7 +43,7 @@ Top-level commands:
43
43
  ### Single-thread run
44
44
 
45
45
  ```bash
46
- uv run threadsense run reddit <reddit-url> \
46
+ uv run threadsense run [<source>] <url> \
47
47
  [--format markdown|html|json] \
48
48
  [--with-summary] \
49
49
  [--summary-required] \
@@ -51,7 +51,9 @@ uv run threadsense run reddit <reddit-url> \
51
51
  [--flat]
52
52
  ```
53
53
 
54
- Example:
54
+ Source can be `reddit`, `hn`, `github-discussions`, `gist`, or omitted for auto-detection.
55
+
56
+ Examples:
55
57
 
56
58
  ```bash
57
59
  uv run threadsense run reddit \
@@ -61,6 +63,12 @@ uv run threadsense run reddit \
61
63
  --summary-required
62
64
  ```
63
65
 
66
+ ```bash
67
+ uv run threadsense run gist \
68
+ "https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f" \
69
+ --format markdown
70
+ ```
71
+
64
72
  ### Topic research across subreddits
65
73
 
66
74
  ```bash
@@ -125,6 +133,20 @@ uv run threadsense fetch hn <hackernews-item-url> [--output path/to/raw.json]
125
133
  uv run threadsense fetch github-discussions <discussion-url> [--output path/to/raw.json]
126
134
  ```
127
135
 
136
+ ### `fetch github-gist`
137
+
138
+ ```bash
139
+ uv run threadsense fetch github-gist <gist-url> [--output path/to/raw.json]
140
+ ```
141
+
142
+ Aliases: `fetch gist`
143
+
144
+ Notes:
145
+
146
+ - Authentication is optional. Public gists work without a token (60 req/hr rate limit).
147
+ - Set `THREADSENSE_GITHUB_GIST_TOKEN` for authenticated access (5,000 req/hr).
148
+ - Comments are paginated at 100/page automatically.
149
+
128
150
  ## `normalize`
129
151
 
130
152
  Normalize one raw artifact into the canonical thread schema.
@@ -147,6 +169,19 @@ uv run threadsense normalize hn --input path/to/raw.json [--output path/to/norma
147
169
  uv run threadsense normalize github-discussions --input path/to/raw.json [--output path/to/normalized.json]
148
170
  ```
149
171
 
172
+ ### `normalize github-gist`
173
+
174
+ ```bash
175
+ uv run threadsense normalize github-gist --input path/to/raw.json [--output path/to/normalized.json]
176
+ ```
177
+
178
+ Aliases: `normalize gist`
179
+
180
+ Notes:
181
+
182
+ - Gist comments are flat — all normalized to `depth=0` with no `parent_comment_id`.
183
+ - Multi-file gists are concatenated into a single body, each file prefixed with its filename.
184
+
150
185
  ## `analyze`
151
186
 
152
187
  Run deterministic analysis for one canonical thread artifact.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "threadsense"
7
- version = "0.2.0"
7
+ version = "0.3.0"
8
8
  description = "Evidence-first discussion intelligence tooling."
9
9
  readme = "README.md"
10
10
  license = { file = "LICENSE" }
@@ -2,4 +2,4 @@ from __future__ import annotations
2
2
 
3
3
  __all__ = ["__version__"]
4
4
 
5
- __version__ = "0.2.0"
5
+ __version__ = "0.3.0"