intentkit 0.6.0.dev6__py3-none-any.whl → 0.6.0.dev8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of intentkit might be problematic. Click here for more details.

intentkit/__init__.py CHANGED
@@ -3,7 +3,7 @@
3
3
  A powerful platform for building AI agents with blockchain and cryptocurrency capabilities.
4
4
  """
5
5
 
6
- __version__ = "0.6.0-dev.6"
6
+ __version__ = "0.6.0-dev.8"
7
7
  __author__ = "hyacinthus"
8
8
  __email__ = "hyacinthus@gmail.com"
9
9
 
intentkit/clients/cdp.py CHANGED
@@ -1,9 +1,12 @@
1
+ import json
2
+ import logging
3
+ import os
1
4
  from typing import Dict, Optional
2
5
 
3
- from cdp import Wallet
6
+ from cdp import EvmServerAccount
4
7
  from coinbase_agentkit import (
5
- CdpWalletProvider,
6
- CdpWalletProviderConfig,
8
+ CdpEvmServerWalletProvider,
9
+ CdpEvmServerWalletProviderConfig,
7
10
  )
8
11
 
9
12
  from intentkit.abstracts.skill import SkillStoreABC
@@ -17,31 +20,80 @@ class CdpClient:
17
20
  def __init__(self, agent_id: str, skill_store: SkillStoreABC) -> None:
18
21
  self._agent_id = agent_id
19
22
  self._skill_store = skill_store
20
- self._wallet_provider: Optional[CdpWalletProvider] = None
21
- self._wallet_provider_config: Optional[CdpWalletProviderConfig] = None
23
+ self._wallet_provider: Optional[CdpEvmServerWalletProvider] = None
24
+ self._wallet_provider_config: Optional[CdpEvmServerWalletProviderConfig] = None
22
25
 
23
- async def get_wallet_provider(self) -> CdpWalletProvider:
26
+ async def get_wallet_provider(self) -> CdpEvmServerWalletProvider:
24
27
  if self._wallet_provider:
25
28
  return self._wallet_provider
26
29
  agent: Agent = await self._skill_store.get_agent_config(self._agent_id)
27
30
  agent_data: AgentData = await self._skill_store.get_agent_data(self._agent_id)
28
31
  network_id = agent.network_id or agent.cdp_network_id
29
- self._wallet_provider_config = CdpWalletProviderConfig(
30
- api_key_name=self._skill_store.get_system_config("cdp_api_key_name"),
31
- api_key_private_key=self._skill_store.get_system_config(
32
- "cdp_api_key_private_key"
33
- ),
32
+
33
+ logger = logging.getLogger(__name__)
34
+
35
+ # Collect credentials from system-config first, then env vars as fallback
36
+ api_key_id = (
37
+ self._skill_store.get_system_config("cdp_api_key_name")
38
+ or os.getenv("CDP_API_KEY_ID")
39
+ or os.getenv("CDP_API_KEY_NAME")
40
+ )
41
+
42
+ api_key_secret = (
43
+ self._skill_store.get_system_config("cdp_api_key_private_key")
44
+ or os.getenv("CDP_API_KEY_SECRET")
45
+ or os.getenv("CDP_API_KEY_PRIVATE_KEY")
46
+ )
47
+
48
+ wallet_secret = self._skill_store.get_system_config(
49
+ "cdp_wallet_secret"
50
+ ) or os.getenv("CDP_WALLET_SECRET")
51
+
52
+ address = None
53
+
54
+ # Attempt to override with any wallet-specific secret stored in wallet_data
55
+ if agent_data.cdp_wallet_data:
56
+ try:
57
+ wallet_data = json.loads(agent_data.cdp_wallet_data)
58
+ # Try to get address from the new format or fallback to old format
59
+ if "default_address_id" in wallet_data:
60
+ address = wallet_data["default_address_id"]
61
+
62
+ # Prefer wallet_secret stored alongside the wallet data if present
63
+ if "wallet_secret" in wallet_data:
64
+ wallet_secret = wallet_data["wallet_secret"]
65
+ elif "account_data" in wallet_data and wallet_data["account_data"]:
66
+ # Some versions may nest the secret inside account_data
67
+ wallet_secret = (
68
+ wallet_data["account_data"].get("wallet_secret")
69
+ or wallet_secret
70
+ )
71
+ except json.JSONDecodeError:
72
+ logger.warning(
73
+ "Invalid JSON in cdp_wallet_data for agent %s", self._agent_id
74
+ )
75
+
76
+ self._wallet_provider_config = CdpEvmServerWalletProviderConfig(
77
+ api_key_id=api_key_id,
78
+ api_key_secret=api_key_secret,
34
79
  network_id=network_id,
35
- wallet_data=agent_data.cdp_wallet_data,
80
+ address=address,
81
+ wallet_secret=wallet_secret,
36
82
  )
37
- self._wallet_provider = CdpWalletProvider(self._wallet_provider_config)
83
+ self._wallet_provider = CdpEvmServerWalletProvider(self._wallet_provider_config)
38
84
  return self._wallet_provider
39
85
 
40
- async def get_wallet(self) -> Wallet:
86
+ async def get_account(self) -> EvmServerAccount:
87
+ """Get the account object from the wallet provider.
88
+
89
+ Returns:
90
+ EvmServerAccount: The account object that can be used for balance checks, transfers, etc.
91
+ """
41
92
  wallet_provider = await self.get_wallet_provider()
42
- return wallet_provider._wallet
93
+ # Access the internal account object
94
+ return wallet_provider._account
43
95
 
44
- async def get_provider_config(self) -> CdpWalletProviderConfig:
96
+ async def get_provider_config(self) -> CdpEvmServerWalletProviderConfig:
45
97
  if not self._wallet_provider_config:
46
98
  await self.get_wallet_provider()
47
99
  return self._wallet_provider_config
@@ -5,10 +5,9 @@ from typing import TypedDict
5
5
  from coinbase_agentkit import (
6
6
  AgentKit,
7
7
  AgentKitConfig,
8
- CdpWalletProvider,
8
+ CdpEvmServerWalletProvider,
9
9
  basename_action_provider,
10
10
  cdp_api_action_provider,
11
- cdp_wallet_action_provider,
12
11
  erc20_action_provider,
13
12
  morpho_action_provider,
14
13
  pyth_action_provider,
@@ -37,10 +36,6 @@ class SkillStates(TypedDict):
37
36
  WalletActionProvider_native_transfer: SkillState
38
37
  CdpApiActionProvider_address_reputation: SkillState
39
38
  CdpApiActionProvider_request_faucet_funds: SkillState
40
- CdpWalletActionProvider_deploy_contract: SkillState
41
- CdpWalletActionProvider_deploy_nft: SkillState
42
- CdpWalletActionProvider_deploy_token: SkillState
43
- CdpWalletActionProvider_trade: SkillState
44
39
  PythActionProvider_fetch_price: SkillState
45
40
  PythActionProvider_fetch_price_feed_id: SkillState
46
41
  BasenameActionProvider_register_basename: SkillState
@@ -97,15 +92,15 @@ async def get_skills(
97
92
 
98
93
  # Initialize CDP client
99
94
  cdp_client: CdpClient = await get_cdp_client(agent_id, store)
100
- cdp_wallet_provider: CdpWalletProvider = await cdp_client.get_wallet_provider()
101
- cdp_provider_config = await cdp_client.get_provider_config()
95
+ cdp_wallet_provider: CdpEvmServerWalletProvider = (
96
+ await cdp_client.get_wallet_provider()
97
+ )
102
98
  agent_kit = AgentKit(
103
99
  AgentKitConfig(
104
100
  wallet_provider=cdp_wallet_provider,
105
101
  action_providers=[
106
102
  wallet_action_provider(),
107
- cdp_api_action_provider(cdp_provider_config),
108
- cdp_wallet_action_provider(cdp_provider_config),
103
+ cdp_api_action_provider(),
109
104
  pyth_action_provider(),
110
105
  basename_action_provider(),
111
106
  erc20_action_provider(),
@@ -121,9 +116,11 @@ async def get_skills(
121
116
  tools = []
122
117
  for skill in available_skills:
123
118
  if skill == "get_balance":
119
+ # Get the account object for the custom GetBalance skill
120
+ account = await cdp_client.get_account()
124
121
  tools.append(
125
122
  GetBalance(
126
- wallet=cdp_wallet_provider._wallet,
123
+ account=account,
127
124
  agent_id=agent_id,
128
125
  skill_store=store,
129
126
  )
@@ -1,9 +1,10 @@
1
1
  from typing import Type
2
2
 
3
- from cdp import Wallet
3
+ from cdp import EvmServerAccount
4
4
  from pydantic import BaseModel, Field
5
5
 
6
6
  from intentkit.abstracts.skill import SkillStoreABC
7
+ from intentkit.clients import get_cdp_client
7
8
  from intentkit.skills.cdp.base import CDPBaseTool
8
9
 
9
10
 
@@ -28,7 +29,7 @@ class GetBalance(CDPBaseTool):
28
29
 
29
30
  agent_id: str
30
31
  skill_store: SkillStoreABC
31
- wallet: Wallet | None = None
32
+ account: EvmServerAccount | None = None
32
33
 
33
34
  name: str = "cdp_get_balance"
34
35
  description: str = (
@@ -48,25 +49,52 @@ class GetBalance(CDPBaseTool):
48
49
  str: A message containing the balance information or error message.
49
50
  """
50
51
  try:
51
- if not self.wallet:
52
- return "Failed to get wallet."
52
+ if not self.account:
53
+ return "Failed to get account."
54
+
55
+ # Get network information from CDP client
56
+ cdp_client = await get_cdp_client(self.agent_id, self.skill_store)
57
+ provider_config = await cdp_client.get_provider_config()
58
+ network_id = provider_config.network_id
59
+
60
+ # Map network_id to the format expected by the API
61
+ network_mapping = {
62
+ "base-mainnet": "base",
63
+ "base-sepolia": "base-sepolia",
64
+ "ethereum": "ethereum",
65
+ "ethereum-mainnet": "ethereum",
66
+ }
67
+ api_network = network_mapping.get(network_id, network_id)
68
+
69
+ # For native ETH balance, use the account's balance directly
70
+ if asset_id.lower() == "eth":
71
+ try:
72
+ # Get native balance using Web3
73
+ balance_wei = await self.account.get_balance()
74
+ balance_eth = balance_wei / (10**18) # Convert from wei to ETH
75
+ return f"ETH balance for account {self.account.address}: {balance_eth} ETH"
76
+ except Exception as e:
77
+ return f"Error getting ETH balance: {e!s}"
78
+
79
+ # For other tokens, try the list_token_balances API
80
+ try:
81
+ # list_token_balances returns all token balances for the account
82
+ token_balances = await self.account.list_token_balances(api_network)
53
83
 
54
- # for each address in the wallet, get the balance for the asset
55
- balances = {}
84
+ # Find the balance for the specific asset
85
+ target_balance = None
86
+ for balance in token_balances:
87
+ if balance.asset_id.lower() == asset_id.lower():
88
+ target_balance = balance
89
+ break
90
+
91
+ if target_balance:
92
+ return f"Balance for {asset_id} in account {self.account.address}: {target_balance.amount} {target_balance.asset_id}"
93
+ else:
94
+ return f"No balance found for asset {asset_id} in account {self.account.address}"
56
95
 
57
- try:
58
- for address in self.wallet.addresses:
59
- balance = address.balance(asset_id)
60
- balances[address.address_id] = balance
61
96
  except Exception as e:
62
- return f"Error getting balance for all addresses in the wallet: {e!s}"
63
-
64
- # Format each balance entry on a new line
65
- balance_lines = [
66
- f" {addr}: {balance}" for addr, balance in balances.items()
67
- ]
68
- formatted_balances = "\n".join(balance_lines)
69
- return f"Balances for wallet {self.wallet.id}:\n{formatted_balances}"
97
+ return f"Error getting balance for account: {e!s}"
70
98
 
71
99
  except Exception as e:
72
100
  return f"Error getting balance: {str(e)}"
@@ -193,70 +193,6 @@
193
193
  "description": "State for WalletActionProvider_native_transfer",
194
194
  "default": "private"
195
195
  },
196
- "CdpWalletActionProvider_trade": {
197
- "type": "string",
198
- "title": "CDP Wallet Trade",
199
- "enum": [
200
- "disabled",
201
- "public",
202
- "private"
203
- ],
204
- "x-enum-title": [
205
- "Disabled",
206
- "Agent Owner + All Users",
207
- "Agent Owner Only"
208
- ],
209
- "description": "State for CdpWalletActionProvider_trade",
210
- "default": "private"
211
- },
212
- "CdpWalletActionProvider_deploy_nft": {
213
- "type": "string",
214
- "title": "Cdp Wallet Deploy Nft",
215
- "enum": [
216
- "disabled",
217
- "public",
218
- "private"
219
- ],
220
- "x-enum-title": [
221
- "Disabled",
222
- "Agent Owner + All Users",
223
- "Agent Owner Only"
224
- ],
225
- "description": "State for CdpWalletActionProvider_deploy_nft",
226
- "default": "disabled"
227
- },
228
- "CdpWalletActionProvider_deploy_token": {
229
- "type": "string",
230
- "title": "CDP Wallet Deploy Token",
231
- "enum": [
232
- "disabled",
233
- "public",
234
- "private"
235
- ],
236
- "x-enum-title": [
237
- "Disabled",
238
- "Agent Owner + All Users",
239
- "Agent Owner Only"
240
- ],
241
- "description": "State for CdpWalletActionProvider_deploy_token",
242
- "default": "disabled"
243
- },
244
- "CdpWalletActionProvider_deploy_contract": {
245
- "type": "string",
246
- "title": "CDP Wallet Deploy Contract",
247
- "enum": [
248
- "disabled",
249
- "public",
250
- "private"
251
- ],
252
- "x-enum-title": [
253
- "Disabled",
254
- "Agent Owner + All Users",
255
- "Agent Owner Only"
256
- ],
257
- "description": "State for CdpWalletActionProvider_deploy_contract",
258
- "default": "disabled"
259
- },
260
196
  "CdpApiActionProvider_request_faucet_funds": {
261
197
  "type": "string",
262
198
  "title": "CDP Request Faucet Funds",
@@ -1,6 +1,7 @@
1
1
  from typing import Optional, Type
2
2
 
3
- from cdp import Wallet
3
+ from cdp import EvmServerAccount
4
+ from coinbase_agentkit import CdpEvmServerWalletProvider
4
5
  from pydantic import BaseModel, Field
5
6
 
6
7
  from intentkit.abstracts.skill import SkillStoreABC
@@ -22,9 +23,31 @@ class EnsoBaseTool(IntentKitSkill):
22
23
  description="The skill store for persisting data"
23
24
  )
24
25
 
25
- async def get_wallet(self, context: SkillContext) -> Optional[Wallet]:
26
+ async def get_account(self, context: SkillContext) -> Optional[EvmServerAccount]:
27
+ """Get the account object from the CDP client.
28
+
29
+ Args:
30
+ context: The skill context containing agent information.
31
+
32
+ Returns:
33
+ Optional[EvmServerAccount]: The account object if available.
34
+ """
35
+ client: CdpClient = await get_cdp_client(context.agent.id, self.skill_store)
36
+ return await client.get_account()
37
+
38
+ async def get_wallet_provider(
39
+ self, context: SkillContext
40
+ ) -> Optional[CdpEvmServerWalletProvider]:
41
+ """Get the wallet provider from the CDP client.
42
+
43
+ Args:
44
+ context: The skill context containing agent information.
45
+
46
+ Returns:
47
+ Optional[CdpEvmServerWalletProvider]: The wallet provider if available.
48
+ """
26
49
  client: CdpClient = await get_cdp_client(context.agent.id, self.skill_store)
27
- return await client.get_wallet()
50
+ return await client.get_wallet_provider()
28
51
 
29
52
  def get_chain_provider(self, context: SkillContext) -> Optional[ChainProvider]:
30
53
  return self.skill_store.get_system_config("chain_provider")
@@ -6,9 +6,7 @@ from langchain_core.runnables import RunnableConfig
6
6
  from pydantic import BaseModel, Field
7
7
 
8
8
  from intentkit.skills.base import SkillContext
9
- from intentkit.skills.enso.abi.route import ABI_ROUTE
10
9
  from intentkit.skills.enso.networks import EnsoGetNetworks
11
- from intentkit.utils.tx import EvmContractWrapper
12
10
 
13
11
  from .base import EnsoBaseTool, base_url, default_chain_id
14
12
 
@@ -188,8 +186,7 @@ class EnsoRouteShortcut(EnsoBaseTool):
188
186
  context: SkillContext = self.context_from_config(config)
189
187
  agent_id = context.agent.id
190
188
  api_token = self.get_api_token(context)
191
- chain_provider = self.get_chain_provider(context)
192
- wallet = await self.get_wallet(context)
189
+ account = await self.get_account(context)
193
190
 
194
191
  async with httpx.AsyncClient() as client:
195
192
  try:
@@ -254,7 +251,7 @@ class EnsoRouteShortcut(EnsoBaseTool):
254
251
  tokenOut=tokenOut,
255
252
  ).model_dump(exclude_none=True)
256
253
 
257
- params["fromAddress"] = wallet.addresses[0].address_id
254
+ params["fromAddress"] = account.address
258
255
 
259
256
  response = await client.get(url, headers=headers, params=params)
260
257
  response.raise_for_status() # Raise HTTPError for non-2xx responses
@@ -268,23 +265,27 @@ class EnsoRouteShortcut(EnsoBaseTool):
268
265
  )
269
266
 
270
267
  if broadcast_requested:
271
- rpc_url = chain_provider.get_chain_config_by_id(chainId).rpc_url
272
- contract = EvmContractWrapper(
273
- rpc_url, ABI_ROUTE, json_dict.get("tx")
274
- )
275
-
276
- fn, fn_args = contract.fn_and_args
277
-
278
- fn_args["amountIn"] = str(fn_args["amountIn"])
279
-
280
- invocation = wallet.invoke_contract(
281
- contract_address=contract.dst_addr,
282
- method=fn.fn_name,
283
- abi=ABI_ROUTE,
284
- args=fn_args,
285
- ).wait()
286
-
287
- res.txHash = invocation.transaction.transaction_hash
268
+ # Use the wallet provider to send the transaction
269
+ wallet_provider = await self.get_wallet_provider(context)
270
+
271
+ # Extract transaction data from the Enso API response
272
+ tx_data = json_dict.get("tx", {})
273
+ if tx_data:
274
+ # Send the transaction using the wallet provider
275
+ tx_hash = wallet_provider.send_transaction(
276
+ {
277
+ "to": tx_data.get("to"),
278
+ "data": tx_data.get("data", "0x"),
279
+ "value": tx_data.get("value", 0),
280
+ }
281
+ )
282
+
283
+ # Wait for transaction confirmation
284
+ wallet_provider.wait_for_transaction_receipt(tx_hash)
285
+ res.txHash = tx_hash
286
+ else:
287
+ # For now, return a placeholder transaction hash if no tx data
288
+ res.txHash = "0x0000000000000000000000000000000000000000000000000000000000000000"
288
289
 
289
290
  return res
290
291
 
@@ -6,9 +6,7 @@ from langchain_core.runnables import RunnableConfig
6
6
  from pydantic import BaseModel, Field
7
7
 
8
8
  from intentkit.skills.base import SkillContext
9
- from intentkit.utils.tx import EvmContractWrapper
10
9
 
11
- from .abi.erc20 import ABI_ERC20
12
10
  from .base import EnsoBaseTool, base_url, default_chain_id
13
11
 
14
12
 
@@ -81,14 +79,14 @@ class EnsoGetWalletBalances(EnsoBaseTool):
81
79
 
82
80
  context: SkillContext = self.context_from_config(config)
83
81
  api_token = self.get_api_token(context)
84
- wallet = await self.get_wallet(context)
82
+ account = await self.get_account(context)
85
83
  headers = {
86
84
  "accept": "application/json",
87
85
  "Authorization": f"Bearer {api_token}",
88
86
  }
89
87
 
90
88
  params = EnsoGetBalancesInput(chainId=chainId).model_dump(exclude_none=True)
91
- params["eoaAddress"] = wallet.addresses[0].address_id
89
+ params["eoaAddress"] = account.address
92
90
  params["useEoa"] = True
93
91
 
94
92
  async with httpx.AsyncClient() as client:
@@ -178,7 +176,7 @@ class EnsoGetWalletApprovals(EnsoBaseTool):
178
176
 
179
177
  context: SkillContext = self.context_from_config(config)
180
178
  api_token = self.get_api_token(context)
181
- wallet = await self.get_wallet(context)
179
+ account = await self.get_account(context)
182
180
 
183
181
  headers = {
184
182
  "accept": "application/json",
@@ -187,7 +185,7 @@ class EnsoGetWalletApprovals(EnsoBaseTool):
187
185
 
188
186
  params = EnsoGetApprovalsInput(
189
187
  chainId=chainId,
190
- fromAddress=wallet.addresses[0].address_id,
188
+ fromAddress=account.address,
191
189
  )
192
190
 
193
191
  if kwargs.get("routingStrategy"):
@@ -318,15 +316,14 @@ class EnsoWalletApprove(EnsoBaseTool):
318
316
  url = f"{base_url}/api/v1/wallet/approve"
319
317
  context: SkillContext = self.context_from_config(config)
320
318
  api_token = self.get_api_token(context)
321
- chain_provider = self.get_chain_provider(context)
322
- wallet = await self.get_wallet(context)
319
+ account = await self.get_account(context)
323
320
 
324
321
  headers = {
325
322
  "accept": "application/json",
326
323
  "Authorization": f"Bearer {api_token}",
327
324
  }
328
325
 
329
- from_address = wallet.addresses[0].address_id
326
+ from_address = account.address
330
327
 
331
328
  params = EnsoWalletApproveInput(
332
329
  tokenAddress=tokenAddress,
@@ -352,20 +349,27 @@ class EnsoWalletApprove(EnsoBaseTool):
352
349
  content = EnsoWalletApproveOutput(**json_dict)
353
350
  artifact = EnsoWalletApproveArtifact(**json_dict)
354
351
 
355
- rpc_url = chain_provider.get_chain_config_by_id(chainId).rpc_url
356
- contract = EvmContractWrapper(rpc_url, ABI_ERC20, artifact.tx)
357
-
358
- fn, fn_args = contract.fn_and_args
359
- fn_args["value"] = str(fn_args["value"])
360
-
361
- invocation = wallet.invoke_contract(
362
- contract_address=contract.dst_addr,
363
- method=fn.fn_name,
364
- abi=ABI_ERC20,
365
- args=fn_args,
366
- ).wait()
367
-
368
- artifact.txHash = invocation.transaction.transaction_hash
352
+ # Use the wallet provider to send the transaction
353
+ wallet_provider = await self.get_wallet_provider(context)
354
+
355
+ # Extract transaction data from the Enso API response
356
+ tx_data = json_dict.get("tx", {})
357
+ if tx_data:
358
+ # Send the transaction using the wallet provider
359
+ tx_hash = wallet_provider.send_transaction(
360
+ {
361
+ "to": tx_data.get("to"),
362
+ "data": tx_data.get("data", "0x"),
363
+ "value": tx_data.get("value", 0),
364
+ }
365
+ )
366
+
367
+ # Wait for transaction confirmation
368
+ wallet_provider.wait_for_transaction_receipt(tx_hash)
369
+ artifact.txHash = tx_hash
370
+ else:
371
+ # For now, return without executing the transaction if no tx data
372
+ artifact.txHash = "0x0000000000000000000000000000000000000000000000000000000000000000"
369
373
 
370
374
  # Return the parsed response
371
375
  return (content, artifact)
@@ -10,12 +10,22 @@ Scrape content from URLs and index into a searchable vector store with configura
10
10
  ### 🔎 `query_indexed_content`
11
11
  Search indexed content using semantic similarity to answer questions and retrieve relevant information.
12
12
 
13
+ ### `website_indexer`
14
+ Index entire websites by discovering and scraping all pages using sitemaps. Automatically finds sitemaps from robots.txt, extracts all URLs, and comprehensively indexes website content.
15
+
16
+ ### `document_indexer`
17
+ Import and index document content directly to the vector database. Perfect for adding content from Google Docs, Notion pages, PDFs, or any other document sources by copy-pasting.
18
+
13
19
  ## Key Features
14
20
 
15
- - **Multi-URL Support**: Scrape up to 10 URLs simultaneously
21
+ - **Multi-URL Support**: Scrape up to 10 URLs simultaneously
22
+ - **Sitemap Discovery**: Automatic sitemap detection from robots.txt with common patterns
23
+ - **Direct Text Input**: Add content directly without web scraping
16
24
  - **Smart Chunking**: Configurable text splitting (100-4000 chars) with overlap
17
25
  - **Vector Search**: FAISS + OpenAI embeddings for semantic retrieval
18
26
  - **Agent Storage**: Persistent, per-agent content indexing
27
+ - **Content Filtering**: Include/exclude URL patterns for targeted scraping
28
+ - **Tagging System**: Organize content with custom tags
19
29
  - **Rate Limiting**: Respectful scraping (0.1-10 req/sec)
20
30
 
21
31
  ## Testing Examples
@@ -39,7 +49,27 @@ Please scrape and index this URL: https://docs.crestal.network/introduction
39
49
  Scrape and index https://docs.crestal.network/introduction with chunk size 500 and overlap 100.
40
50
  ```
41
51
 
42
- ### 3. Content Querying
52
+ ### 3. Complete Website Indexing
53
+
54
+ **Agent Prompt:**
55
+ ```
56
+ Index the entire documentation site at https://docs.crestal.network using its sitemap. Include only pages with '/docs/' and '/guides/' in the URL, exclude '/admin/' pages, and limit to 50 URLs.
57
+ ```
58
+
59
+ ### 4. Document Content Import
60
+
61
+ **Agent Prompt:**
62
+ ```
63
+ I'm going to paste some content from my Google Doc. Please add it to the knowledge base:
64
+
65
+ Title: "Meeting Notes - Q4 Strategy"
66
+ Source: "Google Docs"
67
+ Tags: "meeting, strategy, q4, planning"
68
+
69
+ [Paste your document content here...]
70
+ ```
71
+
72
+ ### 5. Content Querying
43
73
 
44
74
  **Agent Prompt (after indexing):**
45
75
  ```
@@ -75,8 +105,9 @@ curl -X POST "http://localhost:8000/agents/your-agent-id/chat" \
75
105
  ## Dependencies
76
106
 
77
107
  Required packages (add to `pyproject.toml` if missing):
78
- - `langchain-community` - WebBaseLoader
108
+ - `langchain-community` - WebBaseLoader and document processing
79
109
  - `langchain-openai` - Embeddings
80
110
  - `langchain-text-splitters` - Document chunking
81
111
  - `faiss-cpu` - Vector storage
82
- - `beautifulsoup4` - HTML parsing
112
+ - `beautifulsoup4` - HTML parsing
113
+ - `httpx` - Async HTTP client for sitemap discovery
@@ -6,10 +6,12 @@ from typing import TypedDict
6
6
  from intentkit.abstracts.skill import SkillStoreABC
7
7
  from intentkit.skills.base import SkillConfig, SkillOwnerState, SkillState
8
8
  from intentkit.skills.web_scraper.base import WebScraperBaseTool
9
+ from intentkit.skills.web_scraper.document_indexer import DocumentIndexer
9
10
  from intentkit.skills.web_scraper.scrape_and_index import (
10
11
  QueryIndexedContent,
11
12
  ScrapeAndIndex,
12
13
  )
14
+ from intentkit.skills.web_scraper.website_indexer import WebsiteIndexer
13
15
 
14
16
  # Cache skills at the system level, because they are stateless
15
17
  _cache: dict[str, WebScraperBaseTool] = {}
@@ -20,6 +22,8 @@ logger = logging.getLogger(__name__)
20
22
  class SkillStates(TypedDict):
21
23
  scrape_and_index: SkillOwnerState
22
24
  query_indexed_content: SkillState
25
+ website_indexer: SkillOwnerState
26
+ document_indexer: SkillOwnerState
23
27
 
24
28
 
25
29
  class Config(SkillConfig):
@@ -87,6 +91,18 @@ def get_web_scraper_skill(
87
91
  skill_store=store,
88
92
  )
89
93
  return _cache[name]
94
+ elif name == "website_indexer":
95
+ if name not in _cache:
96
+ _cache[name] = WebsiteIndexer(
97
+ skill_store=store,
98
+ )
99
+ return _cache[name]
100
+ elif name == "document_indexer":
101
+ if name not in _cache:
102
+ _cache[name] = DocumentIndexer(
103
+ skill_store=store,
104
+ )
105
+ return _cache[name]
90
106
  else:
91
107
  logger.warning(f"Unknown web scraper skill: {name}")
92
108
  return None