net-sift 0.1.0__tar.gz

net_sift-0.1.0/.editorconfig

@@ -0,0 +1,14 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+
+[*.py]
+indent_size = 4
+
+[*.{md,yml,yaml,toml,json}]
+indent_size = 2

net_sift-0.1.0/.gitattributes

@@ -0,0 +1,4 @@
+* text=auto eol=lf
+*.png binary
+*.jpg binary
+*.gif binary

net_sift-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,30 @@
+name: Bug report
+description: Report a problem with Net-Sift
+labels: [bug]
+body:
+  - type: textarea
+    id: what
+    attributes:
+      label: What happened
+      description: What you did and what went wrong.
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: What you expected
+    validations:
+      required: true
+  - type: textarea
+    id: doctor
+    attributes:
+      label: net-sift doctor output
+      description: Paste the output of `net-sift doctor`. Remove anything sensitive.
+      render: text
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: Output of `net-sift --version`, plus OS and Python version.
+    validations:
+      required: true

net_sift-0.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Security report
+    url: https://github.com/ali-rajabpour/Net-Sift/security/advisories/new
+    about: Please report vulnerabilities privately, not as a public issue.

net_sift-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,18 @@
+name: Feature request
+description: Suggest an idea or a new source for Net-Sift
+labels: [enhancement]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What are you trying to do that Net-Sift does not support yet?
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposal
+      description: What should it do? For a new source, link its public API or access method.
+    validations:
+      required: true

net_sift-0.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,12 @@
+## Summary
+
+<!-- What does this change and why. -->
+
+## Checklist
+
+- [ ] `ruff check net_sift tests` passes
+- [ ] `ruff format --check net_sift tests` passes
+- [ ] `pytest` passes
+- [ ] Tests added or updated for the change (no network in tests)
+- [ ] `CHANGELOG.md` updated under "Unreleased"
+- [ ] Change stays within scope: read-only research, no posting, no auth bypass

net_sift-0.1.0/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: weekly
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly

net_sift-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,41 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        include:
+          - os: macos-latest
+            python-version: "3.12"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Lint
+        run: ruff check net_sift tests
+      - name: Format check
+        run: ruff format --check net_sift tests
+      - name: Test
+        run: pytest --cov=net_sift --cov-report=term-missing

net_sift-0.1.0/.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "27 3 * * 1"
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: github/codeql-action/init@v3
+        with:
+          languages: python
+      - uses: github/codeql-action/analyze@v3

net_sift-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,42 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+      - name: Create release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: |
+          python -m pip install --upgrade pip build
+          python -m build
+      - uses: pypa/gh-action-pypi-publish@release/v1

net_sift-0.1.0/.gitignore

@@ -0,0 +1,24 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+venv/
+
+# net-sift runtime: search sessions are user data, never committed
+.net-sift/
+sessions/
+
+# local env / secrets
+.env
+.env.*
+!.env.example
+
+# session handoffs (local memory, never committed)
+.claude/handoffs/
+
+# OS
+.DS_Store

net_sift-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files

net_sift-0.1.0/CHANGELOG.md

@@ -0,0 +1,29 @@
+# Changelog
+
+All notable changes to this project are documented here. The format is based on
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres
+to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-10-01
+
+### Added
+
+- Coverage-first search engine: keyless sources (Bluesky, Hacker News, GitHub,
+  arXiv, Polymarket, StockTwits, Mastodon, Habr, V2EX, Telegram, Sogou WeChat),
+  a window-bisecting gap-closing driver, per-source near-duplicate dedup, and
+  relevance ranking with head-entity grounding, CJK segmentation, a recency boost,
+  and engagement weighting.
+- Walled-platform access (Twitter/X, Reddit, Instagram, Facebook, Bilibili,
+  Xiaohongshu) through the user's logged-in Chromium browser via OpenCLI, with
+  runtime adapter discovery.
+- MCP server exposing `deep_search`, `resume`, `list_sessions`, `cleanup`,
+  `doctor`, `status`, and `fetch`, all context-lean.
+- Session storage under `~/.net-sift/sessions/` with keep-or-delete confirmation.
+- `net-sift` CLI: `install`, `serve`, `doctor`, `status`, `search`.
+- Installer that registers the MCP server and status bar for Claude Code and Codex.
+- Test suite, Ruff linting, and GitHub Actions CI.
+
+[Unreleased]: https://github.com/ali-rajabpour/Net-Sift/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/ali-rajabpour/Net-Sift/releases/tag/v0.1.0

net_sift-0.1.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,34 @@
+# Code of Conduct
+
+## Our pledge
+
+We as members, contributors, and maintainers pledge to make participation in this
+project a harassment-free experience for everyone, regardless of age, body size,
+visible or invisible disability, ethnicity, sex characteristics, gender identity
+and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our standards
+
+Examples of behavior that contributes to a positive environment:
+
+- Showing empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by our mistakes
+
+Unacceptable behavior includes harassment, insulting or derogatory comments,
+public or private harassment, and publishing others' private information without
+permission.
+
+## Enforcement
+
+Instances of abusive or unacceptable behavior may be reported to the maintainers
+through a private GitHub security advisory or by contacting the repository owner.
+All complaints will be reviewed and investigated promptly and fairly.
+
+## Attribution
+
+This Code of Conduct is adapted from the Contributor Covenant, version 2.1,
+available at https://www.contributor-covenant.org/version/2/1/code_of_conduct/.

net_sift-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,41 @@
+# Contributing
+
+Thanks for your interest in Net-Sift.
+
+## Development setup
+
+```bash
+git clone https://github.com/ali-rajabpour/Net-Sift.git
+cd Net-Sift
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+```
+
+## Before you open a pull request
+
+```bash
+ruff check net_sift tests     # lint
+ruff format net_sift tests    # format
+pytest                        # tests
+```
+
+- Keep the engine dependency-light. New third-party dependencies need a clear
+  reason.
+- Add a test for any non-trivial logic. Tests must not hit the network; use
+  fixtures and temp paths.
+- Each source is a callable `(query, since, until, budget) -> (records, ceiling)`.
+  Follow that contract and emit the record shape from `net_sift.engine.core.rec`.
+- Update `CHANGELOG.md` under "Unreleased".
+
+## Scope
+
+Net-Sift fetches and ranks public content for research. It does not post, comment,
+or perform any write action on any platform, and it does not bypass authentication.
+Keep contributions within that scope.
+
+## Adding a source
+
+- Keyless source: add it to `net_sift/engine/sources.py` and register it in
+  `SOURCES`.
+- Walled platform: it is reached through OpenCLI. If OpenCLI gains an adapter,
+  add the site to `WALLED_SITES` in `net_sift/access/opencli.py`.

net_sift-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ali Rajabpour
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

net_sift-0.1.0/NOTICE

@@ -0,0 +1,14 @@
+Net-Sift
+Copyright (c) 2026 Ali Rajabpour
+
+This product includes code adapted from Agent Reach
+(https://github.com/Panniantong/Agent-Reach), licensed under the MIT License,
+Copyright (c) Panniantong.
+
+Portions adapted from Agent Reach:
+  - net_sift/access/opencli.py        (OpenCLI real-browser backend probing)
+  - net_sift/doctor.py                (source health-check scaffolding)
+  - net_sift/guides/*.md              (per-platform setup guides)
+
+OpenCLI itself (https://github.com/jackwener/opencli) is a separate project by
+its own authors and is invoked as an external tool, not vendored here.

net_sift-0.1.0/PKG-INFO

@@ -0,0 +1,228 @@
+Metadata-Version: 2.5
+Name: net-sift
+Version: 0.1.0
+Summary: Coverage-first social and web deep-search as an MCP server: exhaustive multi-source sweeps, smart ranking, and honest gap accounting, with walled platforms reached through your logged-in Chromium browser.
+Project-URL: Homepage, https://github.com/ali-rajabpour/Net-Sift
+Project-URL: Repository, https://github.com/ali-rajabpour/Net-Sift.git
+Author: Ali Rajabpour
+License-Expression: MIT
+License-File: LICENSE
+License-File: NOTICE
+Keywords: deep-search,mcp,opencli,osint,research,social-search
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: defusedxml>=0.7.1
+Requires-Dist: mcp>=2.2.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Net-Sift
+
+Coverage-first social and web deep-search, delivered as an MCP server.
+
+[![CI](https://github.com/ali-rajabpour/Net-Sift/actions/workflows/ci.yml/badge.svg)](https://github.com/ali-rajabpour/Net-Sift/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-green.svg)](https://www.python.org/)
+[![Linting: Ruff](https://img.shields.io/badge/lint-ruff-261230.svg)](https://github.com/astral-sh/ruff)
+
+Net-Sift sweeps many sources for everything being said about a topic, ranks it,
+removes duplicates, and reports what it could not reach. The gap report is part of
+the answer, not an afterthought. Login-walled platforms are reached through your
+own logged-in Chromium browser, so there is no cookie copying and no private-API
+reverse engineering.
+
+## Table of contents
+
+- [Why Net-Sift](#why-net-sift)
+- [Features](#features)
+- [How it works](#how-it-works)
+- [Install](#install)
+- [Quickstart](#quickstart)
+- [Walled platforms](#walled-platforms)
+- [Sources](#sources)
+- [MCP tools](#mcp-tools)
+- [Configuration](#configuration)
+- [Sessions and privacy](#sessions-and-privacy)
+- [Development](#development)
+- [Contributing](#contributing)
+- [Security](#security)
+- [License](#license)
+- [Acknowledgements](#acknowledgements)
+
+## Why Net-Sift
+
+Most research tools return a handful of links or a synthesized answer. Net-Sift
+returns a corpus with an explicit account of coverage: what was retrieved, what hit
+a ceiling, and what was out of reach. It is built for topic surveys, sentiment
+sweeps, competitor and discourse monitoring, and literature-style scans where a
+single search would under-serve the answer.
+
+## Features
+
+- Many sources in one sweep, keyless by default (Bluesky, Hacker News, GitHub,
+  arXiv, Polymarket, StockTwits, Mastodon, and more).
+- Walled platforms (Twitter/X, Reddit, Instagram, Facebook, Bilibili, Xiaohongshu)
+  through your logged-in Chromium browser via OpenCLI.
+- Relevance ranking with head-entity grounding, CJK-aware tokenization, a recency
+  boost, and engagement weighting.
+- A gap-closing driver that bisects the time window to recover tails a source
+  would otherwise hide.
+- Per-source near-duplicate collapse, so repeats do not inflate the corpus while
+  cross-platform coverage is preserved.
+- Context-lean by design: the agent receives summaries, coverage, gaps, and the
+  top results, never the raw corpus.
+- Saved sessions you can continue later, deleted only on your confirmation.
+
+## How it works
+
+```
+query
+  |
+  v
+sources (keyless)          access (walled, via OpenCLI + your browser)
+  \_________________  _________________/
+                    \/
+         gap-closing driver (bisect on ceiling)
+                    |
+            dedup + ranking
+                    |
+         session on disk  --->  summary + coverage + gaps  --->  agent
+```
+
+Keyless sources call public endpoints directly. Walled platforms are reached by
+shelling to `opencli <site> <command>` against your logged-in browser. Results are
+normalized to one record shape, deduplicated, ranked, and written to a session.
+Only the summary returns to the caller.
+
+## Install
+
+Install from source (not yet on PyPI):
+
+```bash
+uv tool install git+https://github.com/ali-rajabpour/Net-Sift.git
+# or: pipx install git+https://github.com/ali-rajabpour/Net-Sift.git
+```
+
+Then register it with your clients:
+
+```bash
+net-sift install              # register the MCP + status bar, check deps, run doctor
+```
+
+`net-sift install` registers the MCP server and a status bar with Claude Code and
+Codex, checks for Node and OpenCLI, and runs the doctor. Restart your client
+afterward so it picks up the server.
+
+## Quickstart
+
+```bash
+net-sift doctor               # what can be reached right now
+net-sift search "topic" --platforms github,arxiv --max 50
+```
+
+In an MCP client, call `deep_search`:
+
+```
+deep_search(query="post-quantum cryptography adoption", since="2026-01-01")
+```
+
+You get a summary with per-source counts, a coverage map, the gap list, the top
+ranked items, and the corpus path.
+
+## Walled platforms
+
+Twitter/X, Reddit, Instagram, Facebook, Bilibili, and Xiaohongshu need a logged-in
+Chromium browser and OpenCLI. See [net_sift/guides/setup-opencli.md](net_sift/guides/setup-opencli.md).
+Short version:
+
+1. Install Node 20.18.1+ and `@jackwener/opencli` (or OpenCLIApp).
+2. Use a Chromium browser (Chrome, Edge, Brave, Arc, Comet, and so on) with the
+   OpenCLI Browser Bridge extension. Safari and Firefox cannot load it.
+3. Log into the platforms you want in that browser.
+4. Run `net-sift doctor` to confirm.
+
+Desktop only. There is no headless or server path for walled platforms.
+
+## Sources
+
+Full table in [net_sift/guides/sources.md](net_sift/guides/sources.md). Keyless
+sources are on by default; walled sources appear once OpenCLI is connected.
+
+## MCP tools
+
+| Tool | Purpose |
+|------|---------|
+| `deep_search` | run a sweep, return a summary and the corpus path |
+| `resume` | continue an earlier search |
+| `list_sessions` | list saved searches |
+| `cleanup` | delete a saved search (after user confirmation) |
+| `doctor` | what is reachable and how to connect walled platforms |
+| `status` | compact connectivity snapshot |
+| `fetch` | readable text for one page |
+
+## Configuration
+
+| Variable | Effect |
+|----------|--------|
+| `GITHUB_TOKEN` | higher GitHub Search API rate limit |
+| `NET_SIFT_HOME` | session storage location (default `~/.net-sift`) |
+| `NET_SIFT_OPENCLI_BIN` | path to the `opencli` binary if not on `PATH` |
+
+All configuration is read from the environment. Net-Sift stores no credentials.
+
+## Sessions and privacy
+
+Each search is written to `~/.net-sift/sessions/<id>/` as `corpus.jsonl` and
+`meta.json`. Nothing is sent anywhere. After a search, Net-Sift reminds you the
+corpus is kept so you can `resume` it, and deletes it only when you call `cleanup`.
+Walled platforms use your own browser session; Net-Sift never reads or copies your
+cookies.
+
+## Development
+
+```bash
+git clone https://github.com/ali-rajabpour/Net-Sift.git
+cd Net-Sift
+python -m venv .venv && source .venv/bin/activate
+pip install -e ".[dev]"
+ruff check net_sift tests && pytest
+```
+
+See [docs/architecture.md](docs/architecture.md) for the design.
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md). Net-Sift is read-only research: it does not
+post, comment, or bypass authentication, and contributions stay within that scope.
+
+## Security
+
+See [SECURITY.md](SECURITY.md). Report vulnerabilities privately through GitHub.
+
+## License
+
+MIT. See [LICENSE](LICENSE). This project includes code adapted from
+[Agent Reach](https://github.com/Panniantong/Agent-Reach) (MIT); see [NOTICE](NOTICE).
+
+## Acknowledgements
+
+- [OpenCLI](https://github.com/jackwener/opencli) for logged-in browser access.
+- [Agent Reach](https://github.com/Panniantong/Agent-Reach) for the onboarding and
+  doctor patterns.