clearfront 2.6.0__tar.gz → 2.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {clearfront-2.6.0 → clearfront-2.7.1}/PKG-INFO +6 -6
- {clearfront-2.6.0 → clearfront-2.7.1}/README.md +3 -3
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/agent.py +54 -12
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/cli.py +12 -0
- clearfront-2.7.1/clearfront/effort.py +65 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/mcp_server.py +6 -3
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/prompts.py +70 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/repl.py +39 -2
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/web/index.html +87 -43
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/web_server.py +31 -18
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront.egg-info/PKG-INFO +6 -6
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront.egg-info/SOURCES.txt +2 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/pyproject.toml +5 -4
- clearfront-2.7.1/tests/test_effort.py +77 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_repl_first_run.py +1 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/LICENSE +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/__init__.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/brightdata.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/correlation.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/corroboration.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/exceptions.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/extractors.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/json_output.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/multi_target.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/pdf_report.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/pivot.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/regexes.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/serp.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/session_history.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/sponsors.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tool_cache.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/__init__.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/data/NOTICE +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/data/wmn-data-unique.json +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/exceptions.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/generate_dorks.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/scrape_url.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_abuseipdb.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_breach.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_censys.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_crt.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_crypto.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_dns.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_domain.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_dorks_live.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_email.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_emailrep.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_exif.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_exposure.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_footprint.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_github.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_gravatar.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_greynoise.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_harvester.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_hudsonrock.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_ip.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_ip2location.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_maigret.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_paste.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_phone.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_shodan.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_username.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_virustotal.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_wayback.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/search_whois.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/tools/whatsmyname.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront/utils.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront.egg-info/dependency_links.txt +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront.egg-info/entry_points.txt +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront.egg-info/requires.txt +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/clearfront.egg-info/top_level.txt +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/setup.cfg +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_authorized_use.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_brightdata.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_cli_openai.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_cli_output.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_corroboration.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_crt.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_dns.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_exif.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_footprint.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_github_code_search.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_graph_export.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_hudsonrock.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_json_export.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_mcp_authorized_use.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_openai_agent.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_pivot_freetier.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_regexes.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_report_export.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_security.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_serp.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_tool_cache.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_tool_parity.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_tools.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_v2140.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_v240.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_wayback_greynoise.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_web_server.py +0 -0
- {clearfront-2.6.0 → clearfront-2.7.1}/tests/test_web_server_openai.py +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: clearfront
|
|
3
|
-
Version: 2.
|
|
4
|
-
Summary: AI-powered OSINT agent, MCP server, and CLI. Interactive REPL + 30 tools. Anthropic Claude or local Ollama. For authorized security research use only.
|
|
3
|
+
Version: 2.7.1
|
|
4
|
+
Summary: Clearfront OSINT: AI-powered OSINT agent, MCP server, and CLI that maps your digital footprint across 3,400+ public data sources. Interactive REPL + 30 tools. Anthropic Claude or local Ollama. For authorized security research use only.
|
|
5
5
|
Author: Scott Martin Anderson
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/scottmartinanderson/clearfront
|
|
8
8
|
Project-URL: Repository, https://github.com/scottmartinanderson/clearfront
|
|
9
9
|
Project-URL: Bug Tracker, https://github.com/scottmartinanderson/clearfront/issues
|
|
10
|
-
Keywords: osint,security,mcp,cli,ai,agent,anthropic,claude,investigation,recon,cybersecurity
|
|
10
|
+
Keywords: clearfront,clearfront-osint,osint,open-source-intelligence,digital-footprint,security,mcp,cli,ai,agent,anthropic,claude,investigation,recon,cybersecurity,privacy
|
|
11
11
|
Classifier: Development Status :: 5 - Production/Stable
|
|
12
12
|
Classifier: Intended Audience :: Developers
|
|
13
13
|
Classifier: Intended Audience :: Information Technology
|
|
@@ -72,7 +72,7 @@ Dynamic: license-file
|
|
|
72
72
|
<div align="center">
|
|
73
73
|
<h1>Clearfront</h1>
|
|
74
74
|
<p><strong>Open-source AI intelligence on your digital footprint.</strong></p>
|
|
75
|
-
<p>
|
|
75
|
+
<p>Clearfront, also known as Clearfront OSINT, is a free and open-source OSINT agent. Give it an email, username, domain, IP, or name, and its AI security analyst scans 3,400+ public data sources in one sweep, then returns a calibrated report and an interactive evidence graph. Check your own exposure, or investigate an authorized target. It runs locally with your own API keys and sends nothing to us.</p>
|
|
76
76
|
<p>Interactive REPL · CLI · local web console · MCP server. Powered by Anthropic Claude, a local Ollama model, or any OpenAI-compatible endpoint.</p>
|
|
77
77
|
</div>
|
|
78
78
|
|
|
@@ -81,7 +81,7 @@ Dynamic: license-file
|
|
|
81
81
|
[](https://www.python.org/)
|
|
82
82
|
[](LICENSE)
|
|
83
83
|
[](https://modelcontextprotocol.io/)
|
|
84
|
-
[](CHANGELOG.md)
|
|
85
85
|
|
|
86
86
|
**[clearfront.sh](https://clearfront.sh)** · [Disclaimer](./DISCLAIMER.md)
|
|
87
87
|
|
|
@@ -226,7 +226,7 @@ clearfront web
|
|
|
226
226
|
|
|
227
227
|
Browser-based AI chat with streaming tool output, inline result cards, and a light/dark theme toggle. Supports fully local inference via Ollama or any OpenAI-compatible endpoint (no Anthropic API key required when using a local backend).
|
|
228
228
|
|
|
229
|
-
The console runs entirely on your machine and binds to `127.0.0.1` by default. Choose your backend and paste your own key in Settings; your keys and the targets you investigate never touch our servers. Screenshots are in [`media
|
|
229
|
+
The console runs entirely on your machine and binds to `127.0.0.1` by default. Choose your backend and paste your own key in Settings; your keys and the targets you investigate never touch our servers. Screenshots are in [`media/`](media/).
|
|
230
230
|
|
|
231
231
|
### MCP Server
|
|
232
232
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
<h1>Clearfront</h1>
|
|
3
3
|
<p><strong>Open-source AI intelligence on your digital footprint.</strong></p>
|
|
4
|
-
<p>
|
|
4
|
+
<p>Clearfront, also known as Clearfront OSINT, is a free and open-source OSINT agent. Give it an email, username, domain, IP, or name, and its AI security analyst scans 3,400+ public data sources in one sweep, then returns a calibrated report and an interactive evidence graph. Check your own exposure, or investigate an authorized target. It runs locally with your own API keys and sends nothing to us.</p>
|
|
5
5
|
<p>Interactive REPL · CLI · local web console · MCP server. Powered by Anthropic Claude, a local Ollama model, or any OpenAI-compatible endpoint.</p>
|
|
6
6
|
</div>
|
|
7
7
|
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
[](https://www.python.org/)
|
|
11
11
|
[](LICENSE)
|
|
12
12
|
[](https://modelcontextprotocol.io/)
|
|
13
|
-
[](CHANGELOG.md)
|
|
14
14
|
|
|
15
15
|
**[clearfront.sh](https://clearfront.sh)** · [Disclaimer](./DISCLAIMER.md)
|
|
16
16
|
|
|
@@ -155,7 +155,7 @@ clearfront web
|
|
|
155
155
|
|
|
156
156
|
Browser-based AI chat with streaming tool output, inline result cards, and a light/dark theme toggle. Supports fully local inference via Ollama or any OpenAI-compatible endpoint (no Anthropic API key required when using a local backend).
|
|
157
157
|
|
|
158
|
-
The console runs entirely on your machine and binds to `127.0.0.1` by default. Choose your backend and paste your own key in Settings; your keys and the targets you investigate never touch our servers. Screenshots are in [`media
|
|
158
|
+
The console runs entirely on your machine and binds to `127.0.0.1` by default. Choose your backend and paste your own key in Settings; your keys and the targets you investigate never touch our servers. Screenshots are in [`media/`](media/).
|
|
159
159
|
|
|
160
160
|
### MCP Server
|
|
161
161
|
|
|
@@ -57,7 +57,8 @@ from clearfront.tools.search_wayback import run_wayback_osint
|
|
|
57
57
|
from clearfront.tools.search_greynoise import run_greynoise_osint
|
|
58
58
|
from clearfront.tools.search_hudsonrock import run_hudsonrock_osint
|
|
59
59
|
from clearfront.pivot import investigate_graph_for_agent
|
|
60
|
-
from clearfront.prompts import
|
|
60
|
+
from clearfront.prompts import system_prompt_for_effort
|
|
61
|
+
from clearfront import effort as _effort
|
|
61
62
|
|
|
62
63
|
logger = logging.getLogger(__name__)
|
|
63
64
|
|
|
@@ -787,17 +788,23 @@ class OISAgent:
|
|
|
787
788
|
self,
|
|
788
789
|
api_key: str | None = None,
|
|
789
790
|
model: str = "claude-sonnet-4-20250514",
|
|
791
|
+
effort: str = _effort.DEFAULT,
|
|
790
792
|
) -> None:
|
|
791
793
|
self.client = anthropic.AsyncAnthropic(
|
|
792
794
|
api_key=api_key or os.environ.get("ANTHROPIC_API_KEY", "")
|
|
793
795
|
)
|
|
794
796
|
self.model = model
|
|
797
|
+
self.effort = _effort.normalize(effort)
|
|
795
798
|
self.history: list[dict[str, Any]] = []
|
|
796
799
|
|
|
797
800
|
def clear_history(self) -> None:
|
|
798
801
|
"""Reset conversation memory."""
|
|
799
802
|
self.history = []
|
|
800
803
|
|
|
804
|
+
def set_effort(self, effort: str) -> None:
|
|
805
|
+
"""Set the sweep effort for subsequent investigations."""
|
|
806
|
+
self.effort = _effort.normalize(effort)
|
|
807
|
+
|
|
801
808
|
async def run(
|
|
802
809
|
self,
|
|
803
810
|
prompt: str,
|
|
@@ -825,8 +832,17 @@ class OISAgent:
|
|
|
825
832
|
tool_calls=[],
|
|
826
833
|
on_tool_call=on_tool_call,
|
|
827
834
|
)
|
|
835
|
+
# Sweep effort shapes the system prompt (enrichment instruction) and caps
|
|
836
|
+
# the number of tool rounds. When the cap is reached we do not abort; we
|
|
837
|
+
# take one more turn with tools disabled so the model writes its report
|
|
838
|
+
# from the evidence gathered so far.
|
|
839
|
+
system_text = system_prompt_for_effort(self.effort)
|
|
840
|
+
max_rounds = _effort.rounds(self.effort)
|
|
841
|
+
tool_rounds = 0
|
|
828
842
|
try:
|
|
829
843
|
while True:
|
|
844
|
+
tool_rounds += 1
|
|
845
|
+
force_final = tool_rounds > max_rounds
|
|
830
846
|
# Prompt caching: a breakpoint on the (single) system block caches the
|
|
831
847
|
# system prompt + all tool definitions together (tools render before
|
|
832
848
|
# system), so that large stable prefix is re-billed at ~0.1x on every
|
|
@@ -840,11 +856,12 @@ class OISAgent:
|
|
|
840
856
|
system=[
|
|
841
857
|
{
|
|
842
858
|
"type": "text",
|
|
843
|
-
"text":
|
|
859
|
+
"text": system_text,
|
|
844
860
|
"cache_control": {"type": "ephemeral"},
|
|
845
861
|
}
|
|
846
862
|
],
|
|
847
863
|
tools=TOOL_DEFINITIONS, # type: ignore[arg-type]
|
|
864
|
+
tool_choice={"type": "none"} if force_final else {"type": "auto"},
|
|
848
865
|
messages=ctx.messages, # type: ignore[arg-type]
|
|
849
866
|
cache_control={"type": "ephemeral"},
|
|
850
867
|
)
|
|
@@ -911,15 +928,21 @@ class OllamaAgent:
|
|
|
911
928
|
self,
|
|
912
929
|
model: str = "llama3.2",
|
|
913
930
|
host: str = "http://localhost:11434",
|
|
931
|
+
effort: str = _effort.DEFAULT,
|
|
914
932
|
) -> None:
|
|
915
933
|
self.model = model
|
|
916
934
|
self.host = host
|
|
935
|
+
self.effort = _effort.normalize(effort)
|
|
917
936
|
self.history: list[dict[str, Any]] = []
|
|
918
937
|
|
|
919
938
|
def clear_history(self) -> None:
|
|
920
939
|
"""Reset conversation memory."""
|
|
921
940
|
self.history = []
|
|
922
941
|
|
|
942
|
+
def set_effort(self, effort: str) -> None:
|
|
943
|
+
"""Set the sweep effort for subsequent investigations."""
|
|
944
|
+
self.effort = _effort.normalize(effort)
|
|
945
|
+
|
|
923
946
|
async def run(
|
|
924
947
|
self,
|
|
925
948
|
prompt: str,
|
|
@@ -962,21 +985,27 @@ class OllamaAgent:
|
|
|
962
985
|
|
|
963
986
|
self.history.append({"role": "user", "content": prompt})
|
|
964
987
|
messages: list[Any] = [
|
|
965
|
-
{"role": "system", "content":
|
|
988
|
+
{"role": "system", "content": system_prompt_for_effort(self.effort)},
|
|
966
989
|
*self.history,
|
|
967
990
|
]
|
|
968
991
|
ctx = _AgentRunContext(messages=messages, tool_calls=[], on_tool_call=on_tool_call)
|
|
969
992
|
|
|
993
|
+
# Sweep effort caps the tool rounds. Ollama has no reliable tool_choice
|
|
994
|
+
# switch, so on the final turn we call without tools, which forces a
|
|
995
|
+
# text answer from the evidence gathered so far.
|
|
996
|
+
max_rounds = _effort.rounds(self.effort)
|
|
997
|
+
tool_rounds = 0
|
|
970
998
|
try:
|
|
971
999
|
client = ollama.AsyncClient(host=self.host)
|
|
972
1000
|
while True:
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
1001
|
+
tool_rounds += 1
|
|
1002
|
+
force_final = tool_rounds > max_rounds
|
|
1003
|
+
chat_kwargs: dict[str, Any] = {"model": self.model, "messages": ctx.messages}
|
|
1004
|
+
if not force_final:
|
|
1005
|
+
chat_kwargs["tools"] = _OLLAMA_TOOLS
|
|
1006
|
+
response = await client.chat(**chat_kwargs)
|
|
978
1007
|
msg = response.message
|
|
979
|
-
if not msg.tool_calls:
|
|
1008
|
+
if force_final or not msg.tool_calls:
|
|
980
1009
|
text = msg.content or ""
|
|
981
1010
|
self.history.append({"role": "assistant", "content": text})
|
|
982
1011
|
return AgentResponse(content=text, tool_calls=ctx.tool_calls)
|
|
@@ -1079,17 +1108,23 @@ class OpenAICompatibleAgent:
|
|
|
1079
1108
|
model: str = "gpt-4o-mini",
|
|
1080
1109
|
base_url: str = "http://localhost:8080/v1",
|
|
1081
1110
|
api_key: str | None = None,
|
|
1111
|
+
effort: str = _effort.DEFAULT,
|
|
1082
1112
|
) -> None:
|
|
1083
1113
|
self.model = model
|
|
1084
1114
|
self.base_url = base_url
|
|
1085
1115
|
# Many local servers ignore the key, but the SDK requires a non-empty string.
|
|
1086
1116
|
self.api_key = api_key or os.environ.get("OPENAI_API_KEY", "") or "sk-no-key-required"
|
|
1117
|
+
self.effort = _effort.normalize(effort)
|
|
1087
1118
|
self.history: list[dict[str, Any]] = []
|
|
1088
1119
|
|
|
1089
1120
|
def clear_history(self) -> None:
|
|
1090
1121
|
"""Reset conversation memory."""
|
|
1091
1122
|
self.history = []
|
|
1092
1123
|
|
|
1124
|
+
def set_effort(self, effort: str) -> None:
|
|
1125
|
+
"""Set the sweep effort for subsequent investigations."""
|
|
1126
|
+
self.effort = _effort.normalize(effort)
|
|
1127
|
+
|
|
1093
1128
|
async def run(
|
|
1094
1129
|
self,
|
|
1095
1130
|
prompt: str,
|
|
@@ -1124,19 +1159,26 @@ class OpenAICompatibleAgent:
|
|
|
1124
1159
|
|
|
1125
1160
|
self.history.append({"role": "user", "content": prompt})
|
|
1126
1161
|
messages: list[Any] = [
|
|
1127
|
-
{"role": "system", "content":
|
|
1162
|
+
{"role": "system", "content": system_prompt_for_effort(self.effort)},
|
|
1128
1163
|
*self.history,
|
|
1129
1164
|
]
|
|
1130
1165
|
ctx = _AgentRunContext(messages=messages, tool_calls=[], on_tool_call=on_tool_call)
|
|
1131
1166
|
|
|
1167
|
+
# Sweep effort caps the tool rounds. On the final turn we set
|
|
1168
|
+
# tool_choice="none" so the model writes its report instead of calling
|
|
1169
|
+
# another tool.
|
|
1170
|
+
max_rounds = _effort.rounds(self.effort)
|
|
1171
|
+
tool_rounds = 0
|
|
1132
1172
|
try:
|
|
1133
1173
|
client = openai.AsyncOpenAI(base_url=self.base_url, api_key=self.api_key)
|
|
1134
1174
|
while True:
|
|
1175
|
+
tool_rounds += 1
|
|
1176
|
+
force_final = tool_rounds > max_rounds
|
|
1135
1177
|
response = await client.chat.completions.create(
|
|
1136
1178
|
model=self.model,
|
|
1137
1179
|
messages=ctx.messages,
|
|
1138
1180
|
tools=_OPENAI_TOOLS,
|
|
1139
|
-
tool_choice="auto",
|
|
1181
|
+
tool_choice="none" if force_final else "auto",
|
|
1140
1182
|
max_tokens=_MAX_TOKENS,
|
|
1141
1183
|
)
|
|
1142
1184
|
if not response.choices:
|
|
@@ -1148,7 +1190,7 @@ class OpenAICompatibleAgent:
|
|
|
1148
1190
|
),
|
|
1149
1191
|
)
|
|
1150
1192
|
msg = response.choices[0].message
|
|
1151
|
-
if not msg.tool_calls:
|
|
1193
|
+
if force_final or not msg.tool_calls:
|
|
1152
1194
|
text = msg.content or ""
|
|
1153
1195
|
self.history.append({"role": "assistant", "content": text})
|
|
1154
1196
|
return AgentResponse(content=text, tool_calls=ctx.tool_calls)
|
|
@@ -213,6 +213,17 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
213
213
|
dest="is_pdf_disabled",
|
|
214
214
|
help="Disable automatic PDF generation alongside Markdown reports.",
|
|
215
215
|
)
|
|
216
|
+
parser.add_argument(
|
|
217
|
+
"--effort",
|
|
218
|
+
type=str,
|
|
219
|
+
default="deeper",
|
|
220
|
+
choices=["faster", "balanced", "deeper"],
|
|
221
|
+
help=(
|
|
222
|
+
"Sweep effort for the interactive analyst (default: deeper). "
|
|
223
|
+
"faster: fewer sources, quick pass. balanced: main sources. "
|
|
224
|
+
"deeper: follows every lead. Change it live with the 'effort' command."
|
|
225
|
+
),
|
|
226
|
+
)
|
|
216
227
|
|
|
217
228
|
subparsers = parser.add_subparsers(dest="command", metavar="command")
|
|
218
229
|
|
|
@@ -1389,6 +1400,7 @@ async def _async_main() -> None:
|
|
|
1389
1400
|
openai_model=getattr(args, "openai_model", "gpt-4o-mini"),
|
|
1390
1401
|
openai_api_key=getattr(args, "openai_api_key", None),
|
|
1391
1402
|
is_pdf_disabled=getattr(args, "is_pdf_disabled", False),
|
|
1403
|
+
effort=getattr(args, "effort", "deeper"),
|
|
1392
1404
|
)
|
|
1393
1405
|
await repl.run()
|
|
1394
1406
|
return
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# clearfront/effort.py
|
|
2
|
+
"""Sweep effort: shared level metadata and the tool-round ceiling per level.
|
|
3
|
+
|
|
4
|
+
Faster, Balanced, and Deeper trade collection breadth (the number of tool
|
|
5
|
+
rounds the analyst runs) and the matching enrichment instruction, never the
|
|
6
|
+
analyst's reasoning or output quality. Deeper is the full default fan-out; its
|
|
7
|
+
ceiling stays env-tunable via ``OIS_MAX_TOOL_ROUNDS``. The lighter levels are
|
|
8
|
+
capped below it and never exceed it.
|
|
9
|
+
|
|
10
|
+
This module holds the pieces that are identical across surfaces: the level list
|
|
11
|
+
(used by the web console chip and the REPL ``effort`` command) and the per-level
|
|
12
|
+
round ceiling (used by both agent loops). The prompt wording itself is
|
|
13
|
+
surface-specific and lives with each surface: the web console keeps its own
|
|
14
|
+
graph-flavoured enrichment text in ``web_server``; the terminal agent's version
|
|
15
|
+
lives in ``prompts`` (see ``system_prompt_for_effort``).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
|
|
22
|
+
DEFAULT = "deeper"
|
|
23
|
+
|
|
24
|
+
# User-facing labels and descriptions. The descriptions are kept identical to
|
|
25
|
+
# the web console's sweep-effort menu (clearfront/web/index.html effortLevels) so
|
|
26
|
+
# every surface describes the feature the same way.
|
|
27
|
+
LEVELS: list[dict[str, str]] = [
|
|
28
|
+
{
|
|
29
|
+
"v": "faster",
|
|
30
|
+
"name": "Faster",
|
|
31
|
+
"desc": "Checks fewer sources for a quick sweep. Finishes fast and uses less of your API budget.",
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"v": "balanced",
|
|
35
|
+
"name": "Balanced",
|
|
36
|
+
"desc": "Covers the main sources. A middle ground on time and cost.",
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"v": "deeper",
|
|
40
|
+
"name": "Deeper",
|
|
41
|
+
"desc": "Follows every lead for the most complete map, but takes longer and uses more of your API budget.",
|
|
42
|
+
},
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
_VALUES = {level["v"] for level in LEVELS}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def normalize(effort: str | None) -> str:
|
|
49
|
+
"""Return a valid effort value, falling back to the default for anything unknown."""
|
|
50
|
+
return effort if effort in _VALUES else DEFAULT
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def rounds(effort: str) -> int:
|
|
54
|
+
"""Tool-round ceiling for a effort level. Deeper stays env-tunable (default 12);
|
|
55
|
+
the lighter levels are capped below it and never exceed it."""
|
|
56
|
+
ceiling = int(os.environ.get("OIS_MAX_TOOL_ROUNDS", "12"))
|
|
57
|
+
return {"faster": min(4, ceiling), "balanced": min(8, ceiling)}.get(effort, ceiling)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def describe(effort: str) -> str:
|
|
61
|
+
"""One-line description for a effort value (for the REPL 'effort' command)."""
|
|
62
|
+
for level in LEVELS:
|
|
63
|
+
if level["v"] == effort:
|
|
64
|
+
return level["desc"]
|
|
65
|
+
return ""
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
# clearfront/mcp_server.py
|
|
2
2
|
"""
|
|
3
|
-
Clearfront MCP Server, v2.
|
|
3
|
+
Clearfront OSINT MCP Server, v2.7.1
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
Clearfront (Clearfront OSINT) is an open-source AI OSINT agent that maps a
|
|
6
|
+
digital footprint across 3,400+ public data sources and returns a calibrated
|
|
7
|
+
intelligence report. This server exposes all 30 OSINT tool capabilities plus
|
|
8
|
+
multi-target investigation to MCP-compliant AI clients over standard I/O.
|
|
9
|
+
Tools include:
|
|
7
10
|
search_email, search_username, search_maigret, search_breach, search_whois,
|
|
8
11
|
search_ip, search_domain, generate_dorks, search_paste, search_phone,
|
|
9
12
|
search_shodan, search_virustotal, search_censys, search_ip2location,
|
|
@@ -83,6 +83,76 @@ CRITICAL RULES:
|
|
|
83
83
|
SYSTEM_PROMPT = INVESTIGATION_STRATEGY + "\n\n" + ANALYST_CORE
|
|
84
84
|
|
|
85
85
|
|
|
86
|
+
# ---------------------------------------------------------------------------
|
|
87
|
+
# Sweep effort (terminal agent): swap the enrichment instruction and, for the
|
|
88
|
+
# lighter levels, prepend a forceful collection-mode line. Deeper is the full
|
|
89
|
+
# default fan-out and returns SYSTEM_PROMPT byte-identical. The round ceiling
|
|
90
|
+
# that pairs with each level lives in clearfront/effort.py.
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
# The exact aggressive-enrichment sentence embedded in ANALYST_CORE above. Kept
|
|
94
|
+
# as a constant so the lighter levels can swap it out reliably; a test asserts
|
|
95
|
+
# it still occurs verbatim in SYSTEM_PROMPT so an edit to ANALYST_CORE cannot
|
|
96
|
+
# silently break the swap.
|
|
97
|
+
_ENRICH_DEEPER = (
|
|
98
|
+
"- Enrich aggressively before finalising. Every time a pivot surfaces (an email, domain, "
|
|
99
|
+
"company, real name, phone, or an additional handle), expand it with the applicable tools, "
|
|
100
|
+
"then expand the new entities those reveal, chaining outward until the tool budget is reached. "
|
|
101
|
+
"The more real, connected entities you surface, the stronger the report."
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
AGENT_ENRICH = {
|
|
105
|
+
"deeper": _ENRICH_DEEPER,
|
|
106
|
+
"balanced": (
|
|
107
|
+
"- Enrich only the strongest pivots before finalising. When a high-value pivot surfaces "
|
|
108
|
+
"(a confirmed email, domain, real name, or primary handle), expand it once with the "
|
|
109
|
+
"applicable tools. Do not chain outward exhaustively across every lead, even if the user's "
|
|
110
|
+
"own message asks you to map everything or run until the tool budget is reached. Deliver a "
|
|
111
|
+
"solid report on the main footprint."
|
|
112
|
+
),
|
|
113
|
+
"faster": (
|
|
114
|
+
"- Do not enrich outward. This is a single focused pass: run only the highest-signal tools "
|
|
115
|
+
"for the target, and do not scrape URLs or chase company, domain, breach, or secondary "
|
|
116
|
+
"pivots, even if the user's own message asks you to map everything or run until the tool "
|
|
117
|
+
"budget is reached. Deliver a tight, accurate report from what the priority tools return, "
|
|
118
|
+
"and note that a deeper sweep is available for the full map."
|
|
119
|
+
),
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
# Prepended only for the lighter levels, so the instruction to run light wins
|
|
123
|
+
# over any "map everything" language in the base prompt or the user's message.
|
|
124
|
+
AGENT_MODE_LINE = {
|
|
125
|
+
"balanced": (
|
|
126
|
+
"COLLECTION MODE: BALANCED. This overrides any other instruction, including in the user's "
|
|
127
|
+
"own message, to map every reachable data point or run until the tool budget is reached. "
|
|
128
|
+
"Sweep the main sources and expand only the strongest one or two pivots. Do not chain "
|
|
129
|
+
"outward exhaustively."
|
|
130
|
+
),
|
|
131
|
+
"faster": (
|
|
132
|
+
"COLLECTION MODE: FASTER. This overrides any other instruction, including in the user's own "
|
|
133
|
+
"message, to map every reachable data point, enrich every pivot, or run until the tool "
|
|
134
|
+
"budget is reached. Run a single quick pass of only the highest-signal tools for the target "
|
|
135
|
+
"type. Do not scrape URLs and do not chase company, domain, or secondary pivots."
|
|
136
|
+
),
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def system_prompt_for_effort(effort: str) -> str:
|
|
141
|
+
"""Return the terminal agent's system prompt shaped for a sweep effort.
|
|
142
|
+
|
|
143
|
+
Deeper returns SYSTEM_PROMPT unchanged. Balanced and Faster swap the
|
|
144
|
+
enrichment sentence for the lighter instruction and prepend the matching
|
|
145
|
+
collection-mode line.
|
|
146
|
+
"""
|
|
147
|
+
if effort not in AGENT_ENRICH:
|
|
148
|
+
effort = "deeper"
|
|
149
|
+
core = SYSTEM_PROMPT
|
|
150
|
+
if effort != "deeper":
|
|
151
|
+
core = core.replace(_ENRICH_DEEPER, AGENT_ENRICH[effort])
|
|
152
|
+
preamble = AGENT_MODE_LINE.get(effort)
|
|
153
|
+
return f"{preamble}\n\n{core}" if preamble else core
|
|
154
|
+
|
|
155
|
+
|
|
86
156
|
# ---------------------------------------------------------------------------
|
|
87
157
|
# Compact variant for small local models (Ollama)
|
|
88
158
|
# ---------------------------------------------------------------------------
|
|
@@ -31,6 +31,7 @@ from rich.panel import Panel
|
|
|
31
31
|
|
|
32
32
|
from clearfront import __version__
|
|
33
33
|
from clearfront.agent import OllamaAgent, OpenAICompatibleAgent, OISAgent
|
|
34
|
+
from clearfront import effort as _effort
|
|
34
35
|
|
|
35
36
|
logger = logging.getLogger(__name__)
|
|
36
37
|
|
|
@@ -134,6 +135,7 @@ def _print_help() -> None:
|
|
|
134
135
|
"[bold]Commands:[/]",
|
|
135
136
|
"",
|
|
136
137
|
" [#00ff88]<target>[/] Investigate any target (email, username, domain, IP, name)",
|
|
138
|
+
" [#00ff88]effort[/] Set sweep effort (faster, balanced, deeper)",
|
|
137
139
|
" [#00ff88]clear[/] Clear conversation memory",
|
|
138
140
|
" [#00ff88]save[/] Save last report to reports/",
|
|
139
141
|
" [#00ff88]tools[/] List available OSINT tools",
|
|
@@ -216,6 +218,7 @@ def _print_config(
|
|
|
216
218
|
ollama_host: str,
|
|
217
219
|
is_pdf_disabled: bool,
|
|
218
220
|
openai_base_url: str = "",
|
|
221
|
+
effort: str = _effort.DEFAULT,
|
|
219
222
|
) -> None:
|
|
220
223
|
masked = ("*" * 20 + api_key[-6:]) if api_key and len(api_key) > 6 else "not set"
|
|
221
224
|
rows = [
|
|
@@ -229,6 +232,7 @@ def _print_config(
|
|
|
229
232
|
else:
|
|
230
233
|
rows.append(f"[bold]Ollama:[/] {ollama_host}")
|
|
231
234
|
rows += [
|
|
235
|
+
f"[bold]Effort:[/] {effort}",
|
|
232
236
|
"[bold]Reports:[/] ./reports/",
|
|
233
237
|
f"[bold]PDF:[/] {'disabled' if is_pdf_disabled else 'enabled'}",
|
|
234
238
|
]
|
|
@@ -276,6 +280,7 @@ class OISRepl:
|
|
|
276
280
|
openai_model: str = "gpt-4o-mini",
|
|
277
281
|
openai_api_key: str | None = None,
|
|
278
282
|
is_pdf_disabled: bool = False,
|
|
283
|
+
effort: str = _effort.DEFAULT,
|
|
279
284
|
) -> None:
|
|
280
285
|
self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY", "")
|
|
281
286
|
self._provider = provider
|
|
@@ -285,12 +290,14 @@ class OISRepl:
|
|
|
285
290
|
self._openai_model = openai_model
|
|
286
291
|
self._openai_api_key = openai_api_key
|
|
287
292
|
self._is_pdf_disabled = is_pdf_disabled
|
|
293
|
+
self._effort = _effort.normalize(effort)
|
|
288
294
|
|
|
289
295
|
self._agent: OISAgent | OllamaAgent | OpenAICompatibleAgent
|
|
290
296
|
if provider == "ollama":
|
|
291
297
|
self._agent = OllamaAgent(
|
|
292
298
|
model=ollama_model,
|
|
293
299
|
host=ollama_host,
|
|
300
|
+
effort=self._effort,
|
|
294
301
|
)
|
|
295
302
|
self._display_model = ollama_model
|
|
296
303
|
elif provider == "openai":
|
|
@@ -298,10 +305,11 @@ class OISRepl:
|
|
|
298
305
|
model=openai_model,
|
|
299
306
|
base_url=openai_base_url,
|
|
300
307
|
api_key=openai_api_key,
|
|
308
|
+
effort=self._effort,
|
|
301
309
|
)
|
|
302
310
|
self._display_model = openai_model
|
|
303
311
|
else:
|
|
304
|
-
self._agent = OISAgent(api_key=self._api_key)
|
|
312
|
+
self._agent = OISAgent(api_key=self._api_key, effort=self._effort)
|
|
305
313
|
self._display_model = "claude-sonnet-4-20250514"
|
|
306
314
|
|
|
307
315
|
self._last_response: str = ""
|
|
@@ -449,7 +457,7 @@ class OISRepl:
|
|
|
449
457
|
return False
|
|
450
458
|
os.environ["ANTHROPIC_API_KEY"] = key
|
|
451
459
|
self._api_key = key
|
|
452
|
-
self._agent = OISAgent(api_key=key)
|
|
460
|
+
self._agent = OISAgent(api_key=key, effort=self._effort)
|
|
453
461
|
saved = self._persist_key_to_env(key)
|
|
454
462
|
if saved:
|
|
455
463
|
console.print(f"\n [dim]Key saved to {saved} and active for this session.[/]\n")
|
|
@@ -512,9 +520,38 @@ class OISRepl:
|
|
|
512
520
|
self._ollama_host,
|
|
513
521
|
self._is_pdf_disabled,
|
|
514
522
|
self._openai_base_url,
|
|
523
|
+
self._effort,
|
|
515
524
|
)
|
|
516
525
|
continue
|
|
517
526
|
|
|
527
|
+
# 'effort' shows the current level; 'effort <level>' sets it. Only
|
|
528
|
+
# treat 'effort <arg>' as a command when <arg> is a real level, so
|
|
529
|
+
# a genuine query that happens to start with 'effort' still runs.
|
|
530
|
+
effort_parts = user_input.split(maxsplit=1)
|
|
531
|
+
if effort_parts[0].lower() == "effort" and (
|
|
532
|
+
len(effort_parts) == 1
|
|
533
|
+
or effort_parts[1].strip().lower() in {lvl["v"] for lvl in _effort.LEVELS}
|
|
534
|
+
):
|
|
535
|
+
if len(effort_parts) == 2:
|
|
536
|
+
chosen = effort_parts[1].strip().lower()
|
|
537
|
+
self._effort = chosen
|
|
538
|
+
self._agent.set_effort(chosen)
|
|
539
|
+
console.print(
|
|
540
|
+
f" [dim]Sweep effort set to[/] [#00ff88]{chosen}[/][dim]. "
|
|
541
|
+
f"{_effort.describe(chosen)}[/]\n"
|
|
542
|
+
)
|
|
543
|
+
else:
|
|
544
|
+
console.print(f"\n [bold]Sweep effort:[/] [#00ff88]{self._effort}[/]")
|
|
545
|
+
for lvl in _effort.LEVELS:
|
|
546
|
+
marker = "[#00ff88]›[/]" if lvl["v"] == self._effort else " "
|
|
547
|
+
console.print(
|
|
548
|
+
f" {marker} [#00ff88]{lvl['v']:<9}[/][dim]{lvl['desc']}[/]"
|
|
549
|
+
)
|
|
550
|
+
console.print(
|
|
551
|
+
" [dim]Set with: effort faster | balanced | deeper[/]\n"
|
|
552
|
+
)
|
|
553
|
+
continue
|
|
554
|
+
|
|
518
555
|
if user_input.lower() == "save":
|
|
519
556
|
if self._last_response:
|
|
520
557
|
path = _save_report(self._last_response)
|