clearfront 2.5.0__tar.gz → 2.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {clearfront-2.5.0 → clearfront-2.7.0}/PKG-INFO +28 -28
- {clearfront-2.5.0 → clearfront-2.7.0}/README.md +1 -1
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/agent.py +68 -12
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/cli.py +12 -0
- clearfront-2.7.0/clearfront/effort.py +65 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/mcp_server.py +1 -1
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/prompts.py +70 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/repl.py +39 -2
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/web/index.html +72 -10
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/web_server.py +81 -12
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront.egg-info/PKG-INFO +28 -28
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront.egg-info/SOURCES.txt +2 -0
- clearfront-2.7.0/clearfront.egg-info/requires.txt +53 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/pyproject.toml +28 -28
- clearfront-2.7.0/tests/test_effort.py +77 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_repl_first_run.py +1 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_web_server.py +51 -0
- clearfront-2.5.0/clearfront.egg-info/requires.txt +0 -53
- {clearfront-2.5.0 → clearfront-2.7.0}/LICENSE +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/__init__.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/brightdata.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/correlation.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/corroboration.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/exceptions.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/extractors.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/json_output.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/multi_target.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/pdf_report.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/pivot.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/regexes.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/serp.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/session_history.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/sponsors.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tool_cache.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/__init__.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/data/NOTICE +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/data/wmn-data-unique.json +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/exceptions.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/generate_dorks.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/scrape_url.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_abuseipdb.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_breach.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_censys.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_crt.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_crypto.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_dns.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_domain.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_dorks_live.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_email.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_emailrep.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_exif.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_exposure.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_footprint.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_github.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_gravatar.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_greynoise.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_harvester.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_hudsonrock.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_ip.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_ip2location.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_maigret.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_paste.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_phone.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_shodan.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_username.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_virustotal.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_wayback.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/search_whois.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/tools/whatsmyname.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront/utils.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront.egg-info/dependency_links.txt +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront.egg-info/entry_points.txt +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/clearfront.egg-info/top_level.txt +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/setup.cfg +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_authorized_use.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_brightdata.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_cli_openai.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_cli_output.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_corroboration.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_crt.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_dns.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_exif.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_footprint.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_github_code_search.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_graph_export.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_hudsonrock.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_json_export.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_mcp_authorized_use.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_openai_agent.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_pivot_freetier.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_regexes.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_report_export.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_security.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_serp.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_tool_cache.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_tool_parity.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_tools.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_v2140.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_v240.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_wayback_greynoise.py +0 -0
- {clearfront-2.5.0 → clearfront-2.7.0}/tests/test_web_server_openai.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: clearfront
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.7.0
|
|
4
4
|
Summary: AI-powered OSINT agent, MCP server, and CLI. Interactive REPL + 30 tools. Anthropic Claude or local Ollama. For authorized security research use only.
|
|
5
5
|
Author: Scott Martin Anderson
|
|
6
6
|
License-Expression: MIT
|
|
@@ -23,50 +23,50 @@ Requires-Python: >=3.10
|
|
|
23
23
|
Description-Content-Type: text/markdown
|
|
24
24
|
License-File: LICENSE
|
|
25
25
|
Requires-Dist: mcp>=1.28.1
|
|
26
|
-
Requires-Dist: anthropic>=0.
|
|
27
|
-
Requires-Dist: requests>=2.
|
|
26
|
+
Requires-Dist: anthropic>=0.116.0
|
|
27
|
+
Requires-Dist: requests>=2.34.2
|
|
28
28
|
Requires-Dist: python-whois>=0.9.6
|
|
29
|
-
Requires-Dist: rich>=
|
|
30
|
-
Requires-Dist: prompt_toolkit>=3.0.
|
|
31
|
-
Requires-Dist: python-dotenv>=1.
|
|
29
|
+
Requires-Dist: rich>=15.0.0
|
|
30
|
+
Requires-Dist: prompt_toolkit>=3.0.52
|
|
31
|
+
Requires-Dist: python-dotenv>=1.2.2
|
|
32
32
|
Requires-Dist: aiohttp>=3.14.1
|
|
33
33
|
Requires-Dist: dnspython>=2.8.0
|
|
34
|
-
Requires-Dist: ddgs>=9.
|
|
34
|
+
Requires-Dist: ddgs>=9.14.4
|
|
35
35
|
Requires-Dist: fastapi>=0.139.0
|
|
36
|
-
Requires-Dist: uvicorn[standard]>=0.
|
|
37
|
-
Requires-Dist: httpx>=0.
|
|
36
|
+
Requires-Dist: uvicorn[standard]>=0.51.0
|
|
37
|
+
Requires-Dist: httpx>=0.28.1
|
|
38
38
|
Provides-Extra: dev
|
|
39
|
-
Requires-Dist: pytest>=
|
|
40
|
-
Requires-Dist: pytest-asyncio>=0
|
|
39
|
+
Requires-Dist: pytest>=9.1.1; extra == "dev"
|
|
40
|
+
Requires-Dist: pytest-asyncio>=1.4.0; extra == "dev"
|
|
41
41
|
Requires-Dist: pytest-cov>=7.1.0; extra == "dev"
|
|
42
42
|
Requires-Dist: pytest-freezegun>=0.4.2; extra == "dev"
|
|
43
|
-
Requires-Dist: ruff>=0.
|
|
43
|
+
Requires-Dist: ruff>=0.15.20; extra == "dev"
|
|
44
44
|
Provides-Extra: shodan
|
|
45
45
|
Requires-Dist: shodan>=1.31.0; extra == "shodan"
|
|
46
46
|
Provides-Extra: ollama
|
|
47
|
-
Requires-Dist: ollama>=0.
|
|
47
|
+
Requires-Dist: ollama>=0.6.2; extra == "ollama"
|
|
48
48
|
Provides-Extra: openai
|
|
49
|
-
Requires-Dist: openai>=
|
|
49
|
+
Requires-Dist: openai>=2.44.0; extra == "openai"
|
|
50
50
|
Provides-Extra: pdf
|
|
51
|
-
Requires-Dist: reportlab>=
|
|
51
|
+
Requires-Dist: reportlab>=5.0.0; extra == "pdf"
|
|
52
52
|
Provides-Extra: censys
|
|
53
|
-
Requires-Dist: censys-platform>=0.
|
|
53
|
+
Requires-Dist: censys-platform>=0.15.1; extra == "censys"
|
|
54
54
|
Provides-Extra: web
|
|
55
55
|
Requires-Dist: fastapi>=0.139.0; extra == "web"
|
|
56
|
-
Requires-Dist: uvicorn[standard]>=0.
|
|
57
|
-
Requires-Dist: sse-starlette>=
|
|
58
|
-
Requires-Dist: python-dotenv>=1.
|
|
59
|
-
Requires-Dist: anthropic>=0.
|
|
60
|
-
Requires-Dist: playwright>=1.
|
|
56
|
+
Requires-Dist: uvicorn[standard]>=0.51.0; extra == "web"
|
|
57
|
+
Requires-Dist: sse-starlette>=3.4.5; extra == "web"
|
|
58
|
+
Requires-Dist: python-dotenv>=1.2.2; extra == "web"
|
|
59
|
+
Requires-Dist: anthropic>=0.116.0; extra == "web"
|
|
60
|
+
Requires-Dist: playwright>=1.61.0; extra == "web"
|
|
61
61
|
Provides-Extra: all
|
|
62
62
|
Requires-Dist: shodan>=1.31.0; extra == "all"
|
|
63
|
-
Requires-Dist: ollama>=0.
|
|
64
|
-
Requires-Dist: openai>=
|
|
65
|
-
Requires-Dist: reportlab>=
|
|
66
|
-
Requires-Dist: censys-platform>=0.
|
|
63
|
+
Requires-Dist: ollama>=0.6.2; extra == "all"
|
|
64
|
+
Requires-Dist: openai>=2.44.0; extra == "all"
|
|
65
|
+
Requires-Dist: reportlab>=5.0.0; extra == "all"
|
|
66
|
+
Requires-Dist: censys-platform>=0.15.1; extra == "all"
|
|
67
67
|
Requires-Dist: fastapi>=0.139.0; extra == "all"
|
|
68
|
-
Requires-Dist: uvicorn[standard]>=0.
|
|
69
|
-
Requires-Dist: sse-starlette>=
|
|
68
|
+
Requires-Dist: uvicorn[standard]>=0.51.0; extra == "all"
|
|
69
|
+
Requires-Dist: sse-starlette>=3.4.5; extra == "all"
|
|
70
70
|
Dynamic: license-file
|
|
71
71
|
|
|
72
72
|
<div align="center">
|
|
@@ -81,7 +81,7 @@ Dynamic: license-file
|
|
|
81
81
|
[](https://www.python.org/)
|
|
82
82
|
[](LICENSE)
|
|
83
83
|
[](https://modelcontextprotocol.io/)
|
|
84
|
-
[](CHANGELOG.md)
|
|
85
85
|
|
|
86
86
|
**[clearfront.sh](https://clearfront.sh)** · [Disclaimer](./DISCLAIMER.md)
|
|
87
87
|
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
[](https://www.python.org/)
|
|
11
11
|
[](LICENSE)
|
|
12
12
|
[](https://modelcontextprotocol.io/)
|
|
13
|
-
[](CHANGELOG.md)
|
|
14
14
|
|
|
15
15
|
**[clearfront.sh](https://clearfront.sh)** · [Disclaimer](./DISCLAIMER.md)
|
|
16
16
|
|
|
@@ -57,7 +57,8 @@ from clearfront.tools.search_wayback import run_wayback_osint
|
|
|
57
57
|
from clearfront.tools.search_greynoise import run_greynoise_osint
|
|
58
58
|
from clearfront.tools.search_hudsonrock import run_hudsonrock_osint
|
|
59
59
|
from clearfront.pivot import investigate_graph_for_agent
|
|
60
|
-
from clearfront.prompts import
|
|
60
|
+
from clearfront.prompts import system_prompt_for_effort
|
|
61
|
+
from clearfront import effort as _effort
|
|
61
62
|
|
|
62
63
|
logger = logging.getLogger(__name__)
|
|
63
64
|
|
|
@@ -787,17 +788,23 @@ class OISAgent:
|
|
|
787
788
|
self,
|
|
788
789
|
api_key: str | None = None,
|
|
789
790
|
model: str = "claude-sonnet-4-20250514",
|
|
791
|
+
effort: str = _effort.DEFAULT,
|
|
790
792
|
) -> None:
|
|
791
793
|
self.client = anthropic.AsyncAnthropic(
|
|
792
794
|
api_key=api_key or os.environ.get("ANTHROPIC_API_KEY", "")
|
|
793
795
|
)
|
|
794
796
|
self.model = model
|
|
797
|
+
self.effort = _effort.normalize(effort)
|
|
795
798
|
self.history: list[dict[str, Any]] = []
|
|
796
799
|
|
|
797
800
|
def clear_history(self) -> None:
|
|
798
801
|
"""Reset conversation memory."""
|
|
799
802
|
self.history = []
|
|
800
803
|
|
|
804
|
+
def set_effort(self, effort: str) -> None:
|
|
805
|
+
"""Set the sweep effort for subsequent investigations."""
|
|
806
|
+
self.effort = _effort.normalize(effort)
|
|
807
|
+
|
|
801
808
|
async def run(
|
|
802
809
|
self,
|
|
803
810
|
prompt: str,
|
|
@@ -825,14 +832,38 @@ class OISAgent:
|
|
|
825
832
|
tool_calls=[],
|
|
826
833
|
on_tool_call=on_tool_call,
|
|
827
834
|
)
|
|
835
|
+
# Sweep effort shapes the system prompt (enrichment instruction) and caps
|
|
836
|
+
# the number of tool rounds. When the cap is reached we do not abort; we
|
|
837
|
+
# take one more turn with tools disabled so the model writes its report
|
|
838
|
+
# from the evidence gathered so far.
|
|
839
|
+
system_text = system_prompt_for_effort(self.effort)
|
|
840
|
+
max_rounds = _effort.rounds(self.effort)
|
|
841
|
+
tool_rounds = 0
|
|
828
842
|
try:
|
|
829
843
|
while True:
|
|
844
|
+
tool_rounds += 1
|
|
845
|
+
force_final = tool_rounds > max_rounds
|
|
846
|
+
# Prompt caching: a breakpoint on the (single) system block caches the
|
|
847
|
+
# system prompt + all tool definitions together (tools render before
|
|
848
|
+
# system), so that large stable prefix is re-billed at ~0.1x on every
|
|
849
|
+
# round instead of full price. Top-level cache_control auto-places a
|
|
850
|
+
# second breakpoint on the last message block, so the growing tool-output
|
|
851
|
+
# history is cached incrementally across rounds. Caching changes billing
|
|
852
|
+
# only; the model sees identical content and produces identical output.
|
|
830
853
|
response = await self.client.messages.create(
|
|
831
854
|
model=self.model,
|
|
832
855
|
max_tokens=_MAX_TOKENS,
|
|
833
|
-
system=
|
|
856
|
+
system=[
|
|
857
|
+
{
|
|
858
|
+
"type": "text",
|
|
859
|
+
"text": system_text,
|
|
860
|
+
"cache_control": {"type": "ephemeral"},
|
|
861
|
+
}
|
|
862
|
+
],
|
|
834
863
|
tools=TOOL_DEFINITIONS, # type: ignore[arg-type]
|
|
864
|
+
tool_choice={"type": "none"} if force_final else {"type": "auto"},
|
|
835
865
|
messages=ctx.messages, # type: ignore[arg-type]
|
|
866
|
+
cache_control={"type": "ephemeral"},
|
|
836
867
|
)
|
|
837
868
|
if response.stop_reason == "end_turn":
|
|
838
869
|
text = _extract_first_text(response.content)
|
|
@@ -897,15 +928,21 @@ class OllamaAgent:
|
|
|
897
928
|
self,
|
|
898
929
|
model: str = "llama3.2",
|
|
899
930
|
host: str = "http://localhost:11434",
|
|
931
|
+
effort: str = _effort.DEFAULT,
|
|
900
932
|
) -> None:
|
|
901
933
|
self.model = model
|
|
902
934
|
self.host = host
|
|
935
|
+
self.effort = _effort.normalize(effort)
|
|
903
936
|
self.history: list[dict[str, Any]] = []
|
|
904
937
|
|
|
905
938
|
def clear_history(self) -> None:
|
|
906
939
|
"""Reset conversation memory."""
|
|
907
940
|
self.history = []
|
|
908
941
|
|
|
942
|
+
def set_effort(self, effort: str) -> None:
|
|
943
|
+
"""Set the sweep effort for subsequent investigations."""
|
|
944
|
+
self.effort = _effort.normalize(effort)
|
|
945
|
+
|
|
909
946
|
async def run(
|
|
910
947
|
self,
|
|
911
948
|
prompt: str,
|
|
@@ -948,21 +985,27 @@ class OllamaAgent:
|
|
|
948
985
|
|
|
949
986
|
self.history.append({"role": "user", "content": prompt})
|
|
950
987
|
messages: list[Any] = [
|
|
951
|
-
{"role": "system", "content":
|
|
988
|
+
{"role": "system", "content": system_prompt_for_effort(self.effort)},
|
|
952
989
|
*self.history,
|
|
953
990
|
]
|
|
954
991
|
ctx = _AgentRunContext(messages=messages, tool_calls=[], on_tool_call=on_tool_call)
|
|
955
992
|
|
|
993
|
+
# Sweep effort caps the tool rounds. Ollama has no reliable tool_choice
|
|
994
|
+
# switch, so on the final turn we call without tools, which forces a
|
|
995
|
+
# text answer from the evidence gathered so far.
|
|
996
|
+
max_rounds = _effort.rounds(self.effort)
|
|
997
|
+
tool_rounds = 0
|
|
956
998
|
try:
|
|
957
999
|
client = ollama.AsyncClient(host=self.host)
|
|
958
1000
|
while True:
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
1001
|
+
tool_rounds += 1
|
|
1002
|
+
force_final = tool_rounds > max_rounds
|
|
1003
|
+
chat_kwargs: dict[str, Any] = {"model": self.model, "messages": ctx.messages}
|
|
1004
|
+
if not force_final:
|
|
1005
|
+
chat_kwargs["tools"] = _OLLAMA_TOOLS
|
|
1006
|
+
response = await client.chat(**chat_kwargs)
|
|
964
1007
|
msg = response.message
|
|
965
|
-
if not msg.tool_calls:
|
|
1008
|
+
if force_final or not msg.tool_calls:
|
|
966
1009
|
text = msg.content or ""
|
|
967
1010
|
self.history.append({"role": "assistant", "content": text})
|
|
968
1011
|
return AgentResponse(content=text, tool_calls=ctx.tool_calls)
|
|
@@ -1065,17 +1108,23 @@ class OpenAICompatibleAgent:
|
|
|
1065
1108
|
model: str = "gpt-4o-mini",
|
|
1066
1109
|
base_url: str = "http://localhost:8080/v1",
|
|
1067
1110
|
api_key: str | None = None,
|
|
1111
|
+
effort: str = _effort.DEFAULT,
|
|
1068
1112
|
) -> None:
|
|
1069
1113
|
self.model = model
|
|
1070
1114
|
self.base_url = base_url
|
|
1071
1115
|
# Many local servers ignore the key, but the SDK requires a non-empty string.
|
|
1072
1116
|
self.api_key = api_key or os.environ.get("OPENAI_API_KEY", "") or "sk-no-key-required"
|
|
1117
|
+
self.effort = _effort.normalize(effort)
|
|
1073
1118
|
self.history: list[dict[str, Any]] = []
|
|
1074
1119
|
|
|
1075
1120
|
def clear_history(self) -> None:
|
|
1076
1121
|
"""Reset conversation memory."""
|
|
1077
1122
|
self.history = []
|
|
1078
1123
|
|
|
1124
|
+
def set_effort(self, effort: str) -> None:
|
|
1125
|
+
"""Set the sweep effort for subsequent investigations."""
|
|
1126
|
+
self.effort = _effort.normalize(effort)
|
|
1127
|
+
|
|
1079
1128
|
async def run(
|
|
1080
1129
|
self,
|
|
1081
1130
|
prompt: str,
|
|
@@ -1110,19 +1159,26 @@ class OpenAICompatibleAgent:
|
|
|
1110
1159
|
|
|
1111
1160
|
self.history.append({"role": "user", "content": prompt})
|
|
1112
1161
|
messages: list[Any] = [
|
|
1113
|
-
{"role": "system", "content":
|
|
1162
|
+
{"role": "system", "content": system_prompt_for_effort(self.effort)},
|
|
1114
1163
|
*self.history,
|
|
1115
1164
|
]
|
|
1116
1165
|
ctx = _AgentRunContext(messages=messages, tool_calls=[], on_tool_call=on_tool_call)
|
|
1117
1166
|
|
|
1167
|
+
# Sweep effort caps the tool rounds. On the final turn we set
|
|
1168
|
+
# tool_choice="none" so the model writes its report instead of calling
|
|
1169
|
+
# another tool.
|
|
1170
|
+
max_rounds = _effort.rounds(self.effort)
|
|
1171
|
+
tool_rounds = 0
|
|
1118
1172
|
try:
|
|
1119
1173
|
client = openai.AsyncOpenAI(base_url=self.base_url, api_key=self.api_key)
|
|
1120
1174
|
while True:
|
|
1175
|
+
tool_rounds += 1
|
|
1176
|
+
force_final = tool_rounds > max_rounds
|
|
1121
1177
|
response = await client.chat.completions.create(
|
|
1122
1178
|
model=self.model,
|
|
1123
1179
|
messages=ctx.messages,
|
|
1124
1180
|
tools=_OPENAI_TOOLS,
|
|
1125
|
-
tool_choice="auto",
|
|
1181
|
+
tool_choice="none" if force_final else "auto",
|
|
1126
1182
|
max_tokens=_MAX_TOKENS,
|
|
1127
1183
|
)
|
|
1128
1184
|
if not response.choices:
|
|
@@ -1134,7 +1190,7 @@ class OpenAICompatibleAgent:
|
|
|
1134
1190
|
),
|
|
1135
1191
|
)
|
|
1136
1192
|
msg = response.choices[0].message
|
|
1137
|
-
if not msg.tool_calls:
|
|
1193
|
+
if force_final or not msg.tool_calls:
|
|
1138
1194
|
text = msg.content or ""
|
|
1139
1195
|
self.history.append({"role": "assistant", "content": text})
|
|
1140
1196
|
return AgentResponse(content=text, tool_calls=ctx.tool_calls)
|
|
@@ -213,6 +213,17 @@ def _build_parser() -> argparse.ArgumentParser:
|
|
|
213
213
|
dest="is_pdf_disabled",
|
|
214
214
|
help="Disable automatic PDF generation alongside Markdown reports.",
|
|
215
215
|
)
|
|
216
|
+
parser.add_argument(
|
|
217
|
+
"--effort",
|
|
218
|
+
type=str,
|
|
219
|
+
default="deeper",
|
|
220
|
+
choices=["faster", "balanced", "deeper"],
|
|
221
|
+
help=(
|
|
222
|
+
"Sweep effort for the interactive analyst (default: deeper). "
|
|
223
|
+
"faster: fewer sources, quick pass. balanced: main sources. "
|
|
224
|
+
"deeper: follows every lead. Change it live with the 'effort' command."
|
|
225
|
+
),
|
|
226
|
+
)
|
|
216
227
|
|
|
217
228
|
subparsers = parser.add_subparsers(dest="command", metavar="command")
|
|
218
229
|
|
|
@@ -1389,6 +1400,7 @@ async def _async_main() -> None:
|
|
|
1389
1400
|
openai_model=getattr(args, "openai_model", "gpt-4o-mini"),
|
|
1390
1401
|
openai_api_key=getattr(args, "openai_api_key", None),
|
|
1391
1402
|
is_pdf_disabled=getattr(args, "is_pdf_disabled", False),
|
|
1403
|
+
effort=getattr(args, "effort", "deeper"),
|
|
1392
1404
|
)
|
|
1393
1405
|
await repl.run()
|
|
1394
1406
|
return
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# clearfront/effort.py
|
|
2
|
+
"""Sweep effort: shared level metadata and the tool-round ceiling per level.
|
|
3
|
+
|
|
4
|
+
Faster, Balanced, and Deeper trade collection breadth (the number of tool
|
|
5
|
+
rounds the analyst runs) and the matching enrichment instruction, never the
|
|
6
|
+
analyst's reasoning or output quality. Deeper is the full default fan-out; its
|
|
7
|
+
ceiling stays env-tunable via ``OIS_MAX_TOOL_ROUNDS``. The lighter levels are
|
|
8
|
+
capped below it and never exceed it.
|
|
9
|
+
|
|
10
|
+
This module holds the pieces that are identical across surfaces: the level list
|
|
11
|
+
(used by the web console chip and the REPL ``effort`` command) and the per-level
|
|
12
|
+
round ceiling (used by both agent loops). The prompt wording itself is
|
|
13
|
+
surface-specific and lives with each surface: the web console keeps its own
|
|
14
|
+
graph-flavoured enrichment text in ``web_server``; the terminal agent's version
|
|
15
|
+
lives in ``prompts`` (see ``system_prompt_for_effort``).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
|
|
22
|
+
DEFAULT = "deeper"
|
|
23
|
+
|
|
24
|
+
# User-facing labels and descriptions. The descriptions are kept identical to
|
|
25
|
+
# the web console's sweep-effort menu (clearfront/web/index.html effortLevels) so
|
|
26
|
+
# every surface describes the feature the same way.
|
|
27
|
+
LEVELS: list[dict[str, str]] = [
|
|
28
|
+
{
|
|
29
|
+
"v": "faster",
|
|
30
|
+
"name": "Faster",
|
|
31
|
+
"desc": "Checks fewer sources for a quick sweep. Finishes fast and uses less of your API budget.",
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"v": "balanced",
|
|
35
|
+
"name": "Balanced",
|
|
36
|
+
"desc": "Covers the main sources. A middle ground on time and cost.",
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"v": "deeper",
|
|
40
|
+
"name": "Deeper",
|
|
41
|
+
"desc": "Follows every lead for the most complete map, but takes longer and uses more of your API budget.",
|
|
42
|
+
},
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
_VALUES = {level["v"] for level in LEVELS}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def normalize(effort: str | None) -> str:
|
|
49
|
+
"""Return a valid effort value, falling back to the default for anything unknown."""
|
|
50
|
+
return effort if effort in _VALUES else DEFAULT
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def rounds(effort: str) -> int:
|
|
54
|
+
"""Tool-round ceiling for a effort level. Deeper stays env-tunable (default 12);
|
|
55
|
+
the lighter levels are capped below it and never exceed it."""
|
|
56
|
+
ceiling = int(os.environ.get("OIS_MAX_TOOL_ROUNDS", "12"))
|
|
57
|
+
return {"faster": min(4, ceiling), "balanced": min(8, ceiling)}.get(effort, ceiling)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def describe(effort: str) -> str:
|
|
61
|
+
"""One-line description for a effort value (for the REPL 'effort' command)."""
|
|
62
|
+
for level in LEVELS:
|
|
63
|
+
if level["v"] == effort:
|
|
64
|
+
return level["desc"]
|
|
65
|
+
return ""
|
|
@@ -83,6 +83,76 @@ CRITICAL RULES:
|
|
|
83
83
|
SYSTEM_PROMPT = INVESTIGATION_STRATEGY + "\n\n" + ANALYST_CORE
|
|
84
84
|
|
|
85
85
|
|
|
86
|
+
# ---------------------------------------------------------------------------
|
|
87
|
+
# Sweep effort (terminal agent): swap the enrichment instruction and, for the
|
|
88
|
+
# lighter levels, prepend a forceful collection-mode line. Deeper is the full
|
|
89
|
+
# default fan-out and returns SYSTEM_PROMPT byte-identical. The round ceiling
|
|
90
|
+
# that pairs with each level lives in clearfront/effort.py.
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
# The exact aggressive-enrichment sentence embedded in ANALYST_CORE above. Kept
|
|
94
|
+
# as a constant so the lighter levels can swap it out reliably; a test asserts
|
|
95
|
+
# it still occurs verbatim in SYSTEM_PROMPT so an edit to ANALYST_CORE cannot
|
|
96
|
+
# silently break the swap.
|
|
97
|
+
_ENRICH_DEEPER = (
|
|
98
|
+
"- Enrich aggressively before finalising. Every time a pivot surfaces (an email, domain, "
|
|
99
|
+
"company, real name, phone, or an additional handle), expand it with the applicable tools, "
|
|
100
|
+
"then expand the new entities those reveal, chaining outward until the tool budget is reached. "
|
|
101
|
+
"The more real, connected entities you surface, the stronger the report."
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
AGENT_ENRICH = {
|
|
105
|
+
"deeper": _ENRICH_DEEPER,
|
|
106
|
+
"balanced": (
|
|
107
|
+
"- Enrich only the strongest pivots before finalising. When a high-value pivot surfaces "
|
|
108
|
+
"(a confirmed email, domain, real name, or primary handle), expand it once with the "
|
|
109
|
+
"applicable tools. Do not chain outward exhaustively across every lead, even if the user's "
|
|
110
|
+
"own message asks you to map everything or run until the tool budget is reached. Deliver a "
|
|
111
|
+
"solid report on the main footprint."
|
|
112
|
+
),
|
|
113
|
+
"faster": (
|
|
114
|
+
"- Do not enrich outward. This is a single focused pass: run only the highest-signal tools "
|
|
115
|
+
"for the target, and do not scrape URLs or chase company, domain, breach, or secondary "
|
|
116
|
+
"pivots, even if the user's own message asks you to map everything or run until the tool "
|
|
117
|
+
"budget is reached. Deliver a tight, accurate report from what the priority tools return, "
|
|
118
|
+
"and note that a deeper sweep is available for the full map."
|
|
119
|
+
),
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
# Prepended only for the lighter levels, so the instruction to run light wins
|
|
123
|
+
# over any "map everything" language in the base prompt or the user's message.
|
|
124
|
+
AGENT_MODE_LINE = {
|
|
125
|
+
"balanced": (
|
|
126
|
+
"COLLECTION MODE: BALANCED. This overrides any other instruction, including in the user's "
|
|
127
|
+
"own message, to map every reachable data point or run until the tool budget is reached. "
|
|
128
|
+
"Sweep the main sources and expand only the strongest one or two pivots. Do not chain "
|
|
129
|
+
"outward exhaustively."
|
|
130
|
+
),
|
|
131
|
+
"faster": (
|
|
132
|
+
"COLLECTION MODE: FASTER. This overrides any other instruction, including in the user's own "
|
|
133
|
+
"message, to map every reachable data point, enrich every pivot, or run until the tool "
|
|
134
|
+
"budget is reached. Run a single quick pass of only the highest-signal tools for the target "
|
|
135
|
+
"type. Do not scrape URLs and do not chase company, domain, or secondary pivots."
|
|
136
|
+
),
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def system_prompt_for_effort(effort: str) -> str:
|
|
141
|
+
"""Return the terminal agent's system prompt shaped for a sweep effort.
|
|
142
|
+
|
|
143
|
+
Deeper returns SYSTEM_PROMPT unchanged. Balanced and Faster swap the
|
|
144
|
+
enrichment sentence for the lighter instruction and prepend the matching
|
|
145
|
+
collection-mode line.
|
|
146
|
+
"""
|
|
147
|
+
if effort not in AGENT_ENRICH:
|
|
148
|
+
effort = "deeper"
|
|
149
|
+
core = SYSTEM_PROMPT
|
|
150
|
+
if effort != "deeper":
|
|
151
|
+
core = core.replace(_ENRICH_DEEPER, AGENT_ENRICH[effort])
|
|
152
|
+
preamble = AGENT_MODE_LINE.get(effort)
|
|
153
|
+
return f"{preamble}\n\n{core}" if preamble else core
|
|
154
|
+
|
|
155
|
+
|
|
86
156
|
# ---------------------------------------------------------------------------
|
|
87
157
|
# Compact variant for small local models (Ollama)
|
|
88
158
|
# ---------------------------------------------------------------------------
|
|
@@ -31,6 +31,7 @@ from rich.panel import Panel
|
|
|
31
31
|
|
|
32
32
|
from clearfront import __version__
|
|
33
33
|
from clearfront.agent import OllamaAgent, OpenAICompatibleAgent, OISAgent
|
|
34
|
+
from clearfront import effort as _effort
|
|
34
35
|
|
|
35
36
|
logger = logging.getLogger(__name__)
|
|
36
37
|
|
|
@@ -134,6 +135,7 @@ def _print_help() -> None:
|
|
|
134
135
|
"[bold]Commands:[/]",
|
|
135
136
|
"",
|
|
136
137
|
" [#00ff88]<target>[/] Investigate any target (email, username, domain, IP, name)",
|
|
138
|
+
" [#00ff88]effort[/] Set sweep effort (faster, balanced, deeper)",
|
|
137
139
|
" [#00ff88]clear[/] Clear conversation memory",
|
|
138
140
|
" [#00ff88]save[/] Save last report to reports/",
|
|
139
141
|
" [#00ff88]tools[/] List available OSINT tools",
|
|
@@ -216,6 +218,7 @@ def _print_config(
|
|
|
216
218
|
ollama_host: str,
|
|
217
219
|
is_pdf_disabled: bool,
|
|
218
220
|
openai_base_url: str = "",
|
|
221
|
+
effort: str = _effort.DEFAULT,
|
|
219
222
|
) -> None:
|
|
220
223
|
masked = ("*" * 20 + api_key[-6:]) if api_key and len(api_key) > 6 else "not set"
|
|
221
224
|
rows = [
|
|
@@ -229,6 +232,7 @@ def _print_config(
|
|
|
229
232
|
else:
|
|
230
233
|
rows.append(f"[bold]Ollama:[/] {ollama_host}")
|
|
231
234
|
rows += [
|
|
235
|
+
f"[bold]Effort:[/] {effort}",
|
|
232
236
|
"[bold]Reports:[/] ./reports/",
|
|
233
237
|
f"[bold]PDF:[/] {'disabled' if is_pdf_disabled else 'enabled'}",
|
|
234
238
|
]
|
|
@@ -276,6 +280,7 @@ class OISRepl:
|
|
|
276
280
|
openai_model: str = "gpt-4o-mini",
|
|
277
281
|
openai_api_key: str | None = None,
|
|
278
282
|
is_pdf_disabled: bool = False,
|
|
283
|
+
effort: str = _effort.DEFAULT,
|
|
279
284
|
) -> None:
|
|
280
285
|
self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY", "")
|
|
281
286
|
self._provider = provider
|
|
@@ -285,12 +290,14 @@ class OISRepl:
|
|
|
285
290
|
self._openai_model = openai_model
|
|
286
291
|
self._openai_api_key = openai_api_key
|
|
287
292
|
self._is_pdf_disabled = is_pdf_disabled
|
|
293
|
+
self._effort = _effort.normalize(effort)
|
|
288
294
|
|
|
289
295
|
self._agent: OISAgent | OllamaAgent | OpenAICompatibleAgent
|
|
290
296
|
if provider == "ollama":
|
|
291
297
|
self._agent = OllamaAgent(
|
|
292
298
|
model=ollama_model,
|
|
293
299
|
host=ollama_host,
|
|
300
|
+
effort=self._effort,
|
|
294
301
|
)
|
|
295
302
|
self._display_model = ollama_model
|
|
296
303
|
elif provider == "openai":
|
|
@@ -298,10 +305,11 @@ class OISRepl:
|
|
|
298
305
|
model=openai_model,
|
|
299
306
|
base_url=openai_base_url,
|
|
300
307
|
api_key=openai_api_key,
|
|
308
|
+
effort=self._effort,
|
|
301
309
|
)
|
|
302
310
|
self._display_model = openai_model
|
|
303
311
|
else:
|
|
304
|
-
self._agent = OISAgent(api_key=self._api_key)
|
|
312
|
+
self._agent = OISAgent(api_key=self._api_key, effort=self._effort)
|
|
305
313
|
self._display_model = "claude-sonnet-4-20250514"
|
|
306
314
|
|
|
307
315
|
self._last_response: str = ""
|
|
@@ -449,7 +457,7 @@ class OISRepl:
|
|
|
449
457
|
return False
|
|
450
458
|
os.environ["ANTHROPIC_API_KEY"] = key
|
|
451
459
|
self._api_key = key
|
|
452
|
-
self._agent = OISAgent(api_key=key)
|
|
460
|
+
self._agent = OISAgent(api_key=key, effort=self._effort)
|
|
453
461
|
saved = self._persist_key_to_env(key)
|
|
454
462
|
if saved:
|
|
455
463
|
console.print(f"\n [dim]Key saved to {saved} and active for this session.[/]\n")
|
|
@@ -512,9 +520,38 @@ class OISRepl:
|
|
|
512
520
|
self._ollama_host,
|
|
513
521
|
self._is_pdf_disabled,
|
|
514
522
|
self._openai_base_url,
|
|
523
|
+
self._effort,
|
|
515
524
|
)
|
|
516
525
|
continue
|
|
517
526
|
|
|
527
|
+
# 'effort' shows the current level; 'effort <level>' sets it. Only
|
|
528
|
+
# treat 'effort <arg>' as a command when <arg> is a real level, so
|
|
529
|
+
# a genuine query that happens to start with 'effort' still runs.
|
|
530
|
+
effort_parts = user_input.split(maxsplit=1)
|
|
531
|
+
if effort_parts[0].lower() == "effort" and (
|
|
532
|
+
len(effort_parts) == 1
|
|
533
|
+
or effort_parts[1].strip().lower() in {lvl["v"] for lvl in _effort.LEVELS}
|
|
534
|
+
):
|
|
535
|
+
if len(effort_parts) == 2:
|
|
536
|
+
chosen = effort_parts[1].strip().lower()
|
|
537
|
+
self._effort = chosen
|
|
538
|
+
self._agent.set_effort(chosen)
|
|
539
|
+
console.print(
|
|
540
|
+
f" [dim]Sweep effort set to[/] [#00ff88]{chosen}[/][dim]. "
|
|
541
|
+
f"{_effort.describe(chosen)}[/]\n"
|
|
542
|
+
)
|
|
543
|
+
else:
|
|
544
|
+
console.print(f"\n [bold]Sweep effort:[/] [#00ff88]{self._effort}[/]")
|
|
545
|
+
for lvl in _effort.LEVELS:
|
|
546
|
+
marker = "[#00ff88]›[/]" if lvl["v"] == self._effort else " "
|
|
547
|
+
console.print(
|
|
548
|
+
f" {marker} [#00ff88]{lvl['v']:<9}[/][dim]{lvl['desc']}[/]"
|
|
549
|
+
)
|
|
550
|
+
console.print(
|
|
551
|
+
" [dim]Set with: effort faster | balanced | deeper[/]\n"
|
|
552
|
+
)
|
|
553
|
+
continue
|
|
554
|
+
|
|
518
555
|
if user_input.lower() == "save":
|
|
519
556
|
if self._last_response:
|
|
520
557
|
path = _save_report(self._last_response)
|