valyu-js 2.0.2 → 2.1.1

package/README.md

@@ -1,6 +1,6 @@
 # Valyu SDK
 
-**DeepSearch API for AI**
+**Search for AIs**
 
 Valyu's Deepsearch API gives AI the context it needs. Integrate trusted, high-quality public and proprietary sources, with full-text multimodal retrieval.
 
@@ -10,7 +10,7 @@ Get **$10 free credits** for the Valyu API when you sign up at [Valyu](https://p
 
 ## How does it work?
 
-We do all the heavy lifting for you - through a single API we provide:
+We do all the heavy lifting for you - one unified API for all data:
 
 - **Academic & Research Content** - Access millions of scholarly papers and textbooks
 - **Real-time Web Search** - Get the latest information from across the internet  
@@ -36,7 +36,11 @@ const { Valyu } = require('valyu');
 const valyu = new Valyu("your-api-key-here");
 
 const response = await valyu.search(
-  "Implementation details of agentic search-enhanced large reasoning models"
+  "Implementation details of agentic search-enhanced large reasoning models",
+  {
+    maxNumResults: 5,            // Limit to top 5 results
+    maxPrice: 10                  // Maximum price per thousand queries (CPM)
+  }
 );
 
 console.log(response);
@@ -81,7 +85,7 @@ valyu.search(
 | `relevanceThreshold` | `number` | `0.5` | Minimum relevance score for results (0.0-1.0) |
 | `maxPrice` | `number` | `30` | Maximum price per thousand queries in CPM |
 | `includedSources` | `string[]` | `[]` | Specific data sources or URLs to search |
-| `excludeSources` | `string[]` | `[]` | List of URLs/domains to exclude from search results |
+| `excludeSources` | `string[]` | `[]` | Data sources or URLs to exclude from search |
 | `category` | `string` | `null` | Category filter for results |
 | `startDate` | `string` | `null` | Start date filter in YYYY-MM-DD format |
 | `endDate` | `string` | `null` | End date filter in YYYY-MM-DD format |
@@ -126,6 +130,67 @@ Each `SearchResult` contains:
 }
 ```
 
+### Contents Method
+
+The `contents()` method extracts clean, structured content from web pages with optional AI-powered data extraction and summarization. It accepts an array of URLs as the first parameter, followed by optional configuration parameters.
+
+```javascript
+valyu.contents(
+    urls,                                        // Array of URLs to process (max 10)
+    {
+        summary: false,                          // AI processing: false, true, string, or JSON schema
+        extractEffort: "normal",                 // Extraction effort: "normal" or "high"
+        responseLength: "short",                 // Content length control
+        maxPriceDollars: null                    // Maximum cost limit in USD
+    }
+)
+```
+
+### Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `urls` | `string[]` | *required* | Array of URLs to process (maximum 10 URLs per request) |
+| `summary` | `boolean \| string \| object` | `false` | AI summary configuration: `false` (raw content), `true` (auto summary), string (custom instructions), or JSON schema (structured extraction) |
+| `extractEffort` | `string` | `"normal"` | Extraction thoroughness: `"normal"` (fast) or `"high"` (more thorough but slower) |
+| `responseLength` | `string \| number` | `"short"` | Content length per URL: `"short"` (25k chars), `"medium"` (50k), `"large"` (100k), `"max"` (no limit), or custom number |
+| `maxPriceDollars` | `number` | `null` | Maximum cost limit in USD |
+
+### Contents Response Format
+
+The contents method returns a `ContentsResponse` object with the following structure:
+
+```javascript
+{
+    success: boolean,                           // Request success status
+    error: string | null,                       // Error message if any
+    tx_id: string,                              // Transaction ID for tracking
+    urls_requested: number,                     // Total URLs requested
+    urls_processed: number,                     // Successfully processed URLs
+    urls_failed: number,                        // Failed URL count
+    results: ContentResult[],                   // Array of processed results
+    total_cost_dollars: number,                 // Actual cost charged
+    total_characters: number                    // Total characters extracted
+}
+```
+
+Each `ContentResult` contains:
+
+```javascript
+{
+    url: string,                                // Source URL
+    title: string,                              // Page/document title
+    content: string | number,                   // Extracted content
+    length: number,                             // Content length in characters
+    source: string,                             // Data source identifier
+    summary?: string | object,                  // AI-generated summary (if enabled)
+    summary_success?: boolean,                  // Whether summary generation succeeded
+    data_type?: string,                         // Type of data extracted
+    image_url?: Record<string, string>,         // Extracted images
+    citation?: string                           // APA-style citation
+}
+```
+
 ## Examples
 
 ### Basic Search
@@ -135,24 +200,22 @@ const { Valyu } = require('valyu');
 
 const valyu = new Valyu("your-api-key");
 
-// Simple search with no search parameters
-const response = await valyu.search("what is trans-applifying mRNA");
-console.log(response);
+// Simple search across all sources
+const response = await valyu.search("What is machine learning?");
+console.log(`Found ${response.results.length} results`);
 ```
 
 ### Academic Research
 
 ```javascript
-// Deep search over academic papers
+// Search academic papers on arXiv
 const response = await valyu.search(
-  "implementation details of agentic search-enhanced large reasoning models",
+  "transformer architecture improvements",
   {
     searchType: "proprietary",
-    maxNumResults: 10,
-    relevanceThreshold: 0.6,
     includedSources: ["valyu/valyu-arxiv"],
-    category: "agentic RAG",
-    startDate: "2024-12-01"
+    relevanceThreshold: 0.7,
+    maxNumResults: 10
   }
 );
 ```
@@ -160,62 +223,29 @@ const response = await valyu.search(
 ### Web Search with Date Filtering
 
 ```javascript
-// Web search with date range
+// Search recent web content
 const response = await valyu.search(
-  "what are the grok 4 benchmark results",
+  "AI safety developments",
   {
     searchType: "web",
-    maxNumResults: 7,
-    relevanceThreshold: 0.5,
-    startDate: "2025-06-01",
-    endDate: "2025-07-25"
+    startDate: "2024-01-01",
+    endDate: "2024-12-31",
+    maxNumResults: 5
   }
 );
 ```
 
-### Country-Specific Search
+### Hybrid Search
 
 ```javascript
-// Web search with country filtering
+// Search both web and proprietary sources
 const response = await valyu.search(
-  "what is the weather where i am?",
-  {
-    searchType: "web",
-    maxNumResults: 2,
-    countryCode: "UK",
-    responseLength: "short"
-  }
-);
-```
-
-### Search with Source Exclusion
-
-```javascript
-// Hybrid search excluding specific sources
-const response = await valyu.search(
-  "quantum computing applications in cryptography",
+  "quantum computing breakthroughs",
   {
     searchType: "all",
-    maxNumResults: 8,
-    relevanceThreshold: 0.5,
-    maxPrice: 40,
-    excludeSources: ["paperswithcode.com", "wikipedia.org"],
-    responseLength: "large",
-    isToolCall: true
-  }
-);
-```
-
-### Custom Response Length
-
-```javascript
-// Search with custom character count
-const response = await valyu.search(
-  "State of video generation AI models",
-  {
-    maxNumResults: 10,
-    category: "vLLMs",
-    responseLength: 1000  // Limit to 1000 characters per result
+    category: "technology",
+    relevanceThreshold: 0.6,
+    maxPrice: 50
   }
 );
 ```
@@ -240,6 +270,94 @@ if (response.success) {
 }
 ```
 
+### Content Extraction Examples
+
+#### Basic Content Extraction
+
+```javascript
+// Extract raw content from URLs
+const response = await valyu.contents(
+  ["https://techcrunch.com/2025/08/28/anthropic-users-face-a-new-choice-opt-out-or-share-your-data-for-ai-training/"]
+);
+
+if (response.success) {
+    response.results.forEach(result => {
+        console.log(`Title: ${result.title}`);
+        console.log(`Content: ${result.content.substring(0, 500)}...`);
+    });
+}
+```
+
+#### Content with AI Summary
+
+```javascript
+// Extract content with automatic summarization
+const response = await valyu.contents(
+  ["https://docs.python.org/3/tutorial/"],
+  {
+    summary: true,
+    responseLength: "max"
+  }
+);
+
+response.results.forEach(result => {
+    console.log(`Summary: ${result.summary}`);
+});
+```
+
+#### Structured Data Extraction
+
+```javascript
+// Extract structured data using JSON schema
+const companySchema = {
+  type: "object",
+  properties: {
+    company_name: { type: "string" },
+    founded_year: { type: "integer" },
+    key_products: {
+      type: "array",
+      items: { type: "string" },
+      maxItems: 3
+    }
+  }
+};
+
+const response = await valyu.contents(
+  ["https://en.wikipedia.org/wiki/OpenAI"],
+  {
+    summary: companySchema,
+    responseLength: "max"
+  }
+);
+
+if (response.success) {
+    response.results.forEach(result => {
+        if (result.summary) {
+            console.log(`Structured data: ${JSON.stringify(result.summary, null, 2)}`);
+        }
+    });
+}
+```
+
+#### Multiple URLs
+
+```javascript
+// Process multiple URLs with a cost limit
+const response = await valyu.contents(
+  [
+    "https://www.valyu.network/",
+    "https://docs.valyu.network/overview",
+    "https://www.valyu.network/blogs/why-ai-agents-and-llms-struggle-with-search-and-data-access"
+  ],
+  {
+    summary: "Provide key takeaways in bullet points, and write in very emphasised singaporean english"
+  }
+);
+
+console.log(`Processed ${response.urls_processed}/${response.urls_requested} URLs`);
+console.log(`Cost: $${response.total_cost_dollars.toFixed(4)}`);
+```
+
 ## Authentication
 
 Set your API key in one of these ways:
@@ -277,11 +395,20 @@ if (!response.success) {
 The SDK includes full TypeScript support with type definitions for all parameters:
 
 ```typescript
-import { Valyu, SearchOptions, SearchResponse, CountryCode, ResponseLength } from 'valyu';
+import { 
+  Valyu, 
+  SearchOptions, 
+  SearchResponse,
+  ContentsOptions,
+  ContentsResponse,
+  CountryCode, 
+  ResponseLength 
+} from 'valyu';
 
 const valyu = new Valyu("your-api-key");
 
-const options: SearchOptions = {
+// Search API with types
+const searchOptions: SearchOptions = {
     searchType: "proprietary",
     maxNumResults: 10,
     relevanceThreshold: 0.6,
@@ -290,7 +417,20 @@ const options: SearchOptions = {
     responseLength: "medium" as ResponseLength
 };
 
-const response: SearchResponse = await valyu.search("machine learning", options);
+const searchResponse: SearchResponse = await valyu.search("machine learning", searchOptions);
+
+// Contents API with types
+const contentsOptions: ContentsOptions = {
+    summary: true,
+    extractEffort: "high",
+    responseLength: "medium",
+    maxPriceDollars: 0.10
+};
+
+const contentsResponse: ContentsResponse = await valyu.contents(
+    ["https://example.com"],
+    contentsOptions
+);
 ```
 
 ## Backward Compatibility
@@ -321,7 +461,7 @@ const response = await valyu.context(
 
 ## Getting Started
 
-1. Sign up for a free account at [Valyu](https://exchange.valyu.network)
+1. Sign up for a free account at [Valyu](https://platform.valyu.network)
 2. Get your API key from the dashboard  
 3. Install the SDK: `npm install valyu`
 4. Start building with the examples above
@@ -334,10 +474,16 @@ Run the integration tests:
 npm run test:integration
 ```
 
-Run the v2 API examples:
+Run the Search API examples:
+
+```bash
+node examples/search-examples.js
+```
+
+Run the Contents API examples:
 
 ```bash
-node examples/v2-api-examples.js
+node examples/contents-examples.js
 ```
 
 ## Support

package/dist/index.d.mts

@@ -27,6 +27,7 @@ interface SearchOptions {
     endDate?: string;
     countryCode?: CountryCode;
     responseLength?: ResponseLength;
+    fastMode?: boolean;
 }
 interface SearchResponse {
     success: boolean;
@@ -46,6 +47,80 @@ interface FeedbackResponse {
     success: boolean;
     error?: string;
 }
+type ExtractEffort = "normal" | "high" | "auto";
+type ContentResponseLength = "short" | "medium" | "large" | "max" | number;
+interface ContentsOptions {
+    summary?: boolean | string | object;
+    extractEffort?: ExtractEffort;
+    responseLength?: ContentResponseLength;
+    maxPriceDollars?: number;
+}
+interface ContentResult {
+    url: string;
+    title: string;
+    content: string | number;
+    length: number;
+    source: string;
+    summary?: string | object;
+    summary_success?: boolean;
+    data_type?: string;
+    image_url?: Record<string, string>;
+    citation?: string;
+}
+interface ContentsResponse {
+    success: boolean;
+    error?: string | null;
+    tx_id?: string;
+    urls_requested?: number;
+    urls_processed?: number;
+    urls_failed?: number;
+    results?: ContentResult[];
+    total_cost_dollars?: number;
+    total_characters?: number;
+    isProvisioning?: boolean;
+}
+interface AnswerOptions {
+    structuredOutput?: Record<string, any>;
+    systemInstructions?: string;
+    searchType?: SearchType;
+    dataMaxPrice?: number;
+    countryCode?: CountryCode;
+    includedSources?: string[];
+    excludedSources?: string[];
+    startDate?: string;
+    endDate?: string;
+    fastMode?: boolean;
+}
+interface SearchMetadata {
+    tx_ids: string[];
+    number_of_results: number;
+    total_characters: number;
+}
+interface AIUsage {
+    input_tokens: number;
+    output_tokens: number;
+}
+interface Cost {
+    total_deduction_dollars: number;
+    search_deduction_dollars: number;
+    ai_deduction_dollars: number;
+}
+interface AnswerSuccessResponse {
+    success: true;
+    ai_tx_id: string;
+    original_query: string;
+    contents: string | Record<string, any>;
+    data_type: "structured" | "unstructured";
+    search_results: SearchResult[];
+    search_metadata: SearchMetadata;
+    ai_usage: AIUsage;
+    cost: Cost;
+}
+interface AnswerErrorResponse {
+    success: false;
+    error: string;
+}
+type AnswerResponse = AnswerSuccessResponse | AnswerErrorResponse;
 
 declare class Valyu {
     private baseUrl;
@@ -55,6 +130,26 @@ declare class Valyu {
      * Validates date format (YYYY-MM-DD)
      */
     private validateDateFormat;
+    /**
+     * Validates if a string is a valid URL
+     */
+    private validateUrl;
+    /**
+     * Validates if a string is a valid domain (with optional path)
+     */
+    private validateDomain;
+    /**
+     * Validates if a string is a valid dataset identifier (provider/datasetname)
+     */
+    private validateDatasetId;
+    /**
+     * Validates source strings (domains, URLs, or dataset IDs)
+     */
+    private validateSource;
+    /**
+     * Validates an array of source strings
+     */
+    private validateSources;
     /**
      * Search for information using the Valyu DeepSearch API
      * @param query - The search query string
@@ -71,9 +166,38 @@ declare class Valyu {
      * @param options.endDate - End date filter (YYYY-MM-DD format)
      * @param options.countryCode - Country code filter for search results
      * @param options.responseLength - Response content length: "short"/"medium"/"large"/"max" or integer character count
+     * @param options.fastMode - Fast mode for quicker but shorter results (default: false)
      * @returns Promise resolving to search results
      */
     search(query: string, options?: SearchOptions): Promise<SearchResponse>;
+    /**
+     * Extract content from URLs with optional AI processing
+     * @param urls - Array of URLs to process (max 10)
+     * @param options - Content extraction configuration options
+     * @param options.summary - AI summary configuration: false (raw), true (auto), string (custom), or JSON schema
+     * @param options.extractEffort - Extraction thoroughness: "normal" or "high"
+     * @param options.responseLength - Content length per URL
+     * @param options.maxPriceDollars - Maximum cost limit in USD
+     * @returns Promise resolving to content extraction results
+     */
+    contents(urls: string[], options?: ContentsOptions): Promise<ContentsResponse>;
+    /**
+     * Get AI-powered answers using the Valyu Answer API
+     * @param query - The question or query string
+     * @param options - Answer configuration options
+     * @param options.structuredOutput - JSON Schema object for structured responses
+     * @param options.systemInstructions - Custom system-level instructions (max 2000 chars)
+     * @param options.searchType - Type of search: "web", "proprietary", or "all"
+     * @param options.dataMaxPrice - Maximum spend (USD) for data retrieval
+     * @param options.countryCode - Country code filter for search results
+     * @param options.includedSources - List of specific sources to include
+     * @param options.excludedSources - List of URLs/domains to exclude from search results
+     * @param options.startDate - Start date filter (YYYY-MM-DD format)
+     * @param options.endDate - End date filter (YYYY-MM-DD format)
+     * @param options.fastMode - Fast mode for quicker but shorter results (default: false)
+     * @returns Promise resolving to answer response
+     */
+    answer(query: string, options?: AnswerOptions): Promise<AnswerResponse>;
 }
 
-export { type CountryCode, type FeedbackResponse, type FeedbackSentiment, type ResponseLength, type SearchOptions, type SearchResponse, type SearchType, Valyu };
+export { type AIUsage, type AnswerErrorResponse, type AnswerOptions, type AnswerResponse, type AnswerSuccessResponse, type ContentResponseLength, type ContentResult, type ContentsOptions, type ContentsResponse, type Cost, type CountryCode, type ExtractEffort, type FeedbackResponse, type FeedbackSentiment, type ResponseLength, type SearchMetadata, type SearchOptions, type SearchResponse, type SearchType, Valyu };

package/dist/index.d.ts

@@ -27,6 +27,7 @@ interface SearchOptions {
     endDate?: string;
     countryCode?: CountryCode;
     responseLength?: ResponseLength;
+    fastMode?: boolean;
 }
 interface SearchResponse {
     success: boolean;
@@ -46,6 +47,80 @@ interface FeedbackResponse {
     success: boolean;
     error?: string;
 }
+type ExtractEffort = "normal" | "high" | "auto";
+type ContentResponseLength = "short" | "medium" | "large" | "max" | number;
+interface ContentsOptions {
+    summary?: boolean | string | object;
+    extractEffort?: ExtractEffort;
+    responseLength?: ContentResponseLength;
+    maxPriceDollars?: number;
+}
+interface ContentResult {
+    url: string;
+    title: string;
+    content: string | number;
+    length: number;
+    source: string;
+    summary?: string | object;
+    summary_success?: boolean;
+    data_type?: string;
+    image_url?: Record<string, string>;
+    citation?: string;
+}
+interface ContentsResponse {
+    success: boolean;
+    error?: string | null;
+    tx_id?: string;
+    urls_requested?: number;
+    urls_processed?: number;
+    urls_failed?: number;
+    results?: ContentResult[];
+    total_cost_dollars?: number;
+    total_characters?: number;
+    isProvisioning?: boolean;
+}
+interface AnswerOptions {
+    structuredOutput?: Record<string, any>;
+    systemInstructions?: string;
+    searchType?: SearchType;
+    dataMaxPrice?: number;
+    countryCode?: CountryCode;
+    includedSources?: string[];
+    excludedSources?: string[];
+    startDate?: string;
+    endDate?: string;
+    fastMode?: boolean;
+}
+interface SearchMetadata {
+    tx_ids: string[];
+    number_of_results: number;
+    total_characters: number;
+}
+interface AIUsage {
+    input_tokens: number;
+    output_tokens: number;
+}
+interface Cost {
+    total_deduction_dollars: number;
+    search_deduction_dollars: number;
+    ai_deduction_dollars: number;
+}
+interface AnswerSuccessResponse {
+    success: true;
+    ai_tx_id: string;
+    original_query: string;
+    contents: string | Record<string, any>;
+    data_type: "structured" | "unstructured";
+    search_results: SearchResult[];
+    search_metadata: SearchMetadata;
+    ai_usage: AIUsage;
+    cost: Cost;
+}
+interface AnswerErrorResponse {
+    success: false;
+    error: string;
+}
+type AnswerResponse = AnswerSuccessResponse | AnswerErrorResponse;
 
 declare class Valyu {
     private baseUrl;
@@ -55,6 +130,26 @@ declare class Valyu {
      * Validates date format (YYYY-MM-DD)
      */
     private validateDateFormat;
+    /**
+     * Validates if a string is a valid URL
+     */
+    private validateUrl;
+    /**
+     * Validates if a string is a valid domain (with optional path)
+     */
+    private validateDomain;
+    /**
+     * Validates if a string is a valid dataset identifier (provider/datasetname)
+     */
+    private validateDatasetId;
+    /**
+     * Validates source strings (domains, URLs, or dataset IDs)
+     */
+    private validateSource;
+    /**
+     * Validates an array of source strings
+     */
+    private validateSources;
     /**
      * Search for information using the Valyu DeepSearch API
      * @param query - The search query string
@@ -71,9 +166,38 @@ declare class Valyu {
      * @param options.endDate - End date filter (YYYY-MM-DD format)
      * @param options.countryCode - Country code filter for search results
      * @param options.responseLength - Response content length: "short"/"medium"/"large"/"max" or integer character count
+     * @param options.fastMode - Fast mode for quicker but shorter results (default: false)
      * @returns Promise resolving to search results
      */
     search(query: string, options?: SearchOptions): Promise<SearchResponse>;
+    /**
+     * Extract content from URLs with optional AI processing
+     * @param urls - Array of URLs to process (max 10)
+     * @param options - Content extraction configuration options
+     * @param options.summary - AI summary configuration: false (raw), true (auto), string (custom), or JSON schema
+     * @param options.extractEffort - Extraction thoroughness: "normal" or "high"
+     * @param options.responseLength - Content length per URL
+     * @param options.maxPriceDollars - Maximum cost limit in USD
+     * @returns Promise resolving to content extraction results
+     */
+    contents(urls: string[], options?: ContentsOptions): Promise<ContentsResponse>;
+    /**
+     * Get AI-powered answers using the Valyu Answer API
+     * @param query - The question or query string
+     * @param options - Answer configuration options
+     * @param options.structuredOutput - JSON Schema object for structured responses
+     * @param options.systemInstructions - Custom system-level instructions (max 2000 chars)
+     * @param options.searchType - Type of search: "web", "proprietary", or "all"
+     * @param options.dataMaxPrice - Maximum spend (USD) for data retrieval
+     * @param options.countryCode - Country code filter for search results
+     * @param options.includedSources - List of specific sources to include
+     * @param options.excludedSources - List of URLs/domains to exclude from search results
+     * @param options.startDate - Start date filter (YYYY-MM-DD format)
+     * @param options.endDate - End date filter (YYYY-MM-DD format)
+     * @param options.fastMode - Fast mode for quicker but shorter results (default: false)
+     * @returns Promise resolving to answer response
+     */
+    answer(query: string, options?: AnswerOptions): Promise<AnswerResponse>;
 }
 
-export { type CountryCode, type FeedbackResponse, type FeedbackSentiment, type ResponseLength, type SearchOptions, type SearchResponse, type SearchType, Valyu };
+export { type AIUsage, type AnswerErrorResponse, type AnswerOptions, type AnswerResponse, type AnswerSuccessResponse, type ContentResponseLength, type ContentResult, type ContentsOptions, type ContentsResponse, type Cost, type CountryCode, type ExtractEffort, type FeedbackResponse, type FeedbackSentiment, type ResponseLength, type SearchMetadata, type SearchOptions, type SearchResponse, type SearchType, Valyu };