bigdata-smart-batching 0.1.0__py3-none-any.whl

bigdata_smart_batching/smart_batching_config.py

@@ -0,0 +1,39 @@
+"""
+Configuration constants for smart batching.
+
+This module contains all configuration parameters for the smart batching system,
+including API endpoints, limits, time periods, and file paths.
+"""
+
+import os
+
+# API Configuration
+# Use environment variable or default to production API
+API_BASE_URL = os.getenv("BIGDATA_API_BASE_URL", "https://api.bigdata.com")
+COMENTION_ENDPOINT = "/v1/search/co-mentions/entities"
+VOLUME_ENDPOINT = "/v1/search/volume"
+MAX_ENTITIES_PER_QUERY = 1000  # Max entities the API can return in a single response
+MAX_ENTITIES_IN_ANY_OF = 500  # Max entities we can send in any_of filter (API complexity limit)
+MAX_CHUNKS_PER_BASKET = 1000  # Maximum chunks per basket to stay within query limits
+
+# Authentication
+# Note: API_KEY is not required at import time - it's passed to SmartBatchingPlanner
+# This allows the module to be imported without errors
+API_KEY = os.environ.get("BIGDATA_API_KEY")  # Optional at import time
+
+# Time Period Configuration
+# Default date range: 2 years from January 2021 to December 2022
+START_DATE = "2021-01-01"
+END_DATE = "2022-12-31"
+
+# Volume Buckets for basket creation
+# Used to group companies by their chunk volume for efficient batching
+VOLUME_BUCKETS = {
+    "high": (500, float("inf")),  # 500-1000+ chunks
+    "medium": (100, 500),  # 100-500 chunks
+    "low": (1, 100),  # 1-100 chunks
+    "very_low": (0, 1),  # 0 chunks (headlines only, no semantic content)
+}
+
+# File Paths
+UNIVERSE_CSV_PATH = "us_top3000.csv"  # Default universe file (example: US top 3000 companies)

bigdata_smart_batching-0.1.0.dist-info/METADATA

@@ -0,0 +1,231 @@
+Metadata-Version: 2.4
+Name: bigdata-smart-batching
+Version: 0.1.0
+Summary: High-performance semantic search with intelligent company grouping and parallel execution
+License: MIT
+Requires-Python: >=3.11
+Requires-Dist: matplotlib>=3.7
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas>=2.0
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: requests>=2.31.0
+Description-Content-Type: text/markdown
+
+# Smart Batching Search
+
+A high-performance semantic search system that reduces API queries by **67-99%** (varies by topic specificity) through intelligent company grouping and parallel execution.
+
+This module provides a two-step system for efficient semantic search:
+1. **Planning**: Organize search using smart batching and return total expected chunks
+2. **Execution**: Perform search with proportional sampling to preserve distribution
+
+## Key Benefits
+
+- **67-99% Query Reduction**: Search 4,732 companies with only 17-3,699 queries (varies by topic)
+- **Parallel Execution**: Rate-limited concurrent requests with semaphore control
+- **Proportional Sampling**: Retrieve percentage of results while preserving distribution
+- **Production Ready**: Comprehensive error handling, retries, and logging
+- **Scalable**: Efficiently handles universes with 10,000+ companies
+
+## Installation
+
+### Using uv (Recommended)
+
+```bash
+# Clone or navigate to the project
+cd bigdata-smart-batching
+
+# Install in development mode
+uv sync
+
+# Or with dev dependencies
+uv sync --all-extras
+```
+
+### Environment Setup
+
+Set up environment variables:
+
+```bash
+export BIGDATA_API_KEY="your_api_key_here"
+export BIGDATA_API_BASE_URL="https://api.bigdata.com"  # Optional, defaults to this
+```
+
+Or create a `.env` file:
+
+```
+BIGDATA_API_KEY=your_api_key_here
+BIGDATA_API_BASE_URL=https://api.bigdata.com
+```
+
+## Quick Start
+
+```python
+from bigdata_smart_batching import (
+    plan_search,
+    execute_search,
+    deduplicate_documents,
+    convert_to_dataframe,
+)
+
+# Step 1: Plan the search
+plan = plan_search(
+    text="earnings revenue profit",
+    universe_csv_path="id_name_mapping_us_top_3000.csv",
+    start_date="2023-01-01",
+    end_date="2023-12-31",
+    api_key="your_api_key",  # or set BIGDATA_API_KEY env var
+)
+
+print(f"Total expected chunks: {plan['total_expected_chunks']:,}")
+
+# Step 2: Execute search with 10% of total chunks (preserves distribution)
+results_raw = execute_search(
+    search_plan=plan,
+    chunk_percentage=0.1,
+    requests_per_minute=100,
+)
+
+# Step 3: Deduplicate and convert to DataFrame
+results = deduplicate_documents(results_raw)
+print(f"Retrieved {len(results)} documents (deduplicated)")
+
+df = convert_to_dataframe(results)  # one row per chunk
+```
+
+### Save and Load Plans
+
+```python
+from bigdata_smart_batching import plan_search, execute_search, save_plan, load_plan
+
+# Create and save a plan
+plan = plan_search(
+    text="merger acquisition",
+    universe_csv_path="id_name_mapping_us_top_3000.csv",
+    start_date="2023-01-01",
+    end_date="2023-12-31",
+)
+save_plan(plan, "my_search_plan.json")
+
+# Later: reload and run with different sampling
+plan = load_plan("my_search_plan.json")
+raw_10 = execute_search(plan, chunk_percentage=0.1)
+raw_50 = execute_search(plan, chunk_percentage=0.5)
+```
+
+## How It Works
+
+### Architecture Overview
+
+```
+Step 1: PLANNING
+  Universe CSV  -->  Co-mention API Query  -->  Basket Creation  -->  Search Plan
+
+Step 2: EXECUTION
+  Proportional Sampling  -->  Parallel Search (Rate Limited)  -->  Collect & Aggregate
+```
+
+### Planning (`plan_search()`)
+
+1. Loads the universe of companies from CSV
+2. Queries the comention endpoint to get chunk volumes per company
+3. Splits date ranges by volume when a company exceeds the chunk limit
+4. Creates optimized baskets grouped by volume
+5. Returns a plan with total expected chunks and basket configurations
+
+### Execution (`execute_search()`)
+
+1. Calculates proportional chunks per basket
+2. Ensures minimum of 1 chunk per basket (if expected > 0)
+3. Executes searches in parallel with rate limiting and semaphore
+4. Collects and returns document results
+
+## API Reference
+
+### `plan_search()`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `text` | `str` | required | Search query text |
+| `universe_csv_path` | `str` | required | Path to CSV with entity IDs |
+| `start_date` | `str` | required | Start date (YYYY-MM-DD) |
+| `end_date` | `str` | required | End date (YYYY-MM-DD) |
+| `api_key` | `str` | env var | API key |
+| `api_base_url` | `str` | env var | API base URL |
+| `volume_query_mode` | `str` | `"three_pass"` | `"three_pass"` or `"iterative"` |
+| `apply_volume_splits` | `bool` | `True` | Use volume time series for period splitting |
+| `min_period_days` | `int` | `30` | Minimum days per sub-period |
+
+### `execute_search()`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `search_plan` | `Dict` | required | Plan from `plan_search()` |
+| `chunk_percentage` | `float` | required | 0.0 to 1.0 sampling ratio |
+| `requests_per_minute` | `int` | `100` | Rate limit |
+| `api_key` | `str` | env var | API key |
+| `max_workers` | `int` | `40` | Parallel workers |
+
+### Helper Functions
+
+- **`deduplicate_documents(documents)`** -- Merges duplicate documents by `id`
+- **`load_universe_from_csv(csv_path)`** -- Loads entity IDs from CSV
+- **`convert_to_dataframe(raw_results)`** -- Converts documents to DataFrame (one row per chunk)
+- **`save_plan(plan, path)`** / **`load_plan(path)`** -- Persist plans as JSON
+- **`portfolio_backtesting_pipeline(...)`** -- Long-short portfolio backtesting
+
+## Testing
+
+```bash
+# Run all tests
+uv run pytest
+
+# With coverage
+uv run pytest --cov=bigdata_smart_batching --cov-report=term-missing
+
+# Specific test file
+uv run pytest tests/test_validation.py -v
+```
+
+## Project Structure
+
+```
+bigdata-smart-batching/
+├── pyproject.toml
+├── README.md
+├── .python-version
+├── src/
+│   └── bigdata_smart_batching/
+│       ├── __init__.py
+│       ├── smart_batching.py
+│       ├── smart_batching_config.py
+│       ├── search_function.py
+│       ├── output_converter.py
+│       └── portfolio_backtesting.py
+└── tests/
+    ├── __init__.py
+    ├── test_config.py
+    ├── test_output_converter.py
+    ├── test_validation.py
+    └── test_rate_limiter.py
+```
+
+## Configuration
+
+### Environment Variables
+
+- `BIGDATA_API_KEY`: Required -- Your Bigdata API key
+- `BIGDATA_API_BASE_URL`: Optional -- API base URL (default: `https://api.bigdata.com`)
+
+### Default Settings
+
+- `requests_per_minute`: 100
+- `max_workers`: 40
+- `max_chunks_per_basket`: 1000
+- `volume_query_mode`: `"three_pass"`
+
+## License
+
+This project is part of the Bigdata.com and WorldQuant Challenge.
+
+**Disclaimer**: This software is provided "as is" without warranty of any kind, express or implied. The authors and contributors assume no responsibility for the accuracy, completeness, or usefulness of any information, results, or processes provided. This software is for educational and research purposes only and is not intended to be used as financial advice.

bigdata_smart_batching-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+bigdata_smart_batching/__init__.py,sha256=t19V51mbiS-gO5IJFC1p2Jk3FdplKfUFvWmGOrtAAAs,620
+bigdata_smart_batching/output_converter.py,sha256=_YRlgtkw79nD2OzIKnISD4GEowOZrmizxDgSBGuUhBw,3427
+bigdata_smart_batching/search_function.py,sha256=gdmIq1SpQIKFRCwIoRRXY6TnEHc-mBlRrcmd5VVHwug,41743
+bigdata_smart_batching/smart_batching.py,sha256=5eECKFXyMeT94c4o37wFWAV1-2wCzbB6gsLmMFZh67w,69119
+bigdata_smart_batching/smart_batching_config.py,sha256=wIRmKfUed3TgSEaEO3JIx66NIHwZwd9q_kw3eY_2oSM,1543
+bigdata_smart_batching-0.1.0.dist-info/METADATA,sha256=J4nwxKtwSFOE4_zTyDQDizWZFBOn5bUfh2GjWAxWTFQ,7169
+bigdata_smart_batching-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+bigdata_smart_batching-0.1.0.dist-info/RECORD,,

bigdata_smart_batching-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any