bigdata-smart-batching 1.1.0__tar.gz → 1.2.1__tar.gz

{bigdata_smart_batching-1.1.0 → bigdata_smart_batching-1.2.1}/.gitignore

@@ -110,6 +110,3 @@ Thumbs.db
 *.csv
 
 benchmark/
-benchmark.py
-scatter_date_relevance.py
-simple.py

{bigdata_smart_batching-1.1.0 → bigdata_smart_batching-1.2.1}/CHANGELOG.md

@@ -5,6 +5,28 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.1] - 2026-05-12
+
+### Fixed
+
+- Modified README.md file to add latest changes
+
+## [1.2.0] - 2026-05-12
+
+### Added
+
+- Document-level **sentiment** filters: inclusive `min` / `max` range(s), validated with `validate_sentiment_range`, modeled as `SentimentRange` / `SentimentRangeInput`, serialized to `query.filters.sentiment.ranges` on co-mention, volume, and search payloads. `plan_search(..., sentiment=...)` and `sentiment_filter` plumbed through `SmartBatchingPlanner` co-mention and volume flows. Types and validator re-exported from the package root.
+- **Planning throughput controls**: `plan_search(..., requests_per_minute=..., max_workers=...)` and `SmartBatchingPlanner(..., requests_per_minute=..., max_workers=..., rate_limiter=..., concurrency_limiter=...)` so all planner HTTP work shares one sliding-window rate limiter and one concurrency semaphore across internal thread pools.
+- **`exponential_backoff`** helper in `search_function` (used by retry paths; capped via `MAX_RETRY_BACKOFF_SECONDS`).
+
+### Changed
+
+- **Retries:** co-mention / volume POST retries now use **shared exponential backoff** for `429`, `403`, timeouts, and other failures, instead of parsing `Retry-After` for `429` and separate ad-hoc `403` handling (constant renamed/clamped to `MAX_RETRY_BACKOFF_SECONDS`).
+- **Volume time-series merge:** when aggregating batch results per date, a **`sentiment`** value on each point is the arithmetic mean of batch sentiments for that date (batches without sentiment for that date are omitted from the mean).
+- **`SlidingWindowRateLimiter`:** window accounting aligned with **ceil-based** `max_per_window` expectations (tests updated for low rates and 450 rpm-style cases).
+
+
+
 ## [1.1.0] - 2026-04-20
 
 ### Added

{bigdata_smart_batching-1.1.0 → bigdata_smart_batching-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bigdata-smart-batching
-Version: 1.1.0
+Version: 1.2.1
 Summary: High-performance semantic search with intelligent company grouping and parallel execution
 Project-URL: homepage, https://bigdata.com/api
 Author-email: "Bigdata.com" <support@ravenpack.com>
@@ -29,6 +29,8 @@ This module provides a two-step pipeline for efficient semantic search:
 - **Parallel execution** -- rate-limited concurrent requests with semaphore-controlled concurrency.
 - **Proportional sampling** -- retrieve a percentage of results while preserving the distribution across baskets.
 - **Category filtering** -- restrict searches to specific document categories (news, transcripts, filings, ...).
+- **Sentiment filtering** -- restrict searches to one or more inclusive document-level sentiment ranges.
+- **Throughput controls** -- shared sliding-window rate limiter and concurrency semaphore across all planning HTTP work, with shared exponential backoff on `429` / `403` / timeout retries.
 - **Deterministic plans** -- every request stays within service limits; plans can be saved and replayed.
 - **Scalable** -- handles universes with 10,000+ companies efficiently.
 
@@ -162,6 +164,41 @@ Valid category values are exposed via `VALID_CATEGORY_VALUES` and include:
 `news_public`, `podcasts`, `research`, `research_academic_journals`,
 `research_investment_research`, `transcripts`.
 
+### Sentiment Filtering
+
+Restrict searches to documents whose sentiment falls inside one or more
+inclusive `min` / `max` ranges using the `sentiment` argument. It accepts a
+single `SentimentRange` / dict, or a list of them (multiple disjoint ranges).
+The filter is emitted as `query.filters.sentiment.ranges` on co-mention,
+volume, and search payloads.
+
+```python
+from bigdata_smart_batching import SentimentRange, plan_search
+
+plan = plan_search(
+    universe="id_name_mapping_us_top_3000.csv",
+    start_date="2023-01-01",
+    end_date="2023-12-31",
+    text="earnings revenue profit",
+    sentiment=SentimentRange(min=0.3, max=1.0),
+)
+
+plan = plan_search(
+    universe="id_name_mapping_us_top_3000.csv",
+    start_date="2023-01-01",
+    end_date="2023-12-31",
+    text="earnings revenue profit",
+    sentiment=[
+        {"min": -1.0, "max": -0.5},
+        {"min": 0.5, "max": 1.0},
+    ],
+)
+```
+
+Each range must satisfy `min <= max`. Use
+`validate_sentiment_range(sentiment)` to validate and normalize input into the
+list-of-dicts shape the API expects.
+
 ### Save and Load Plans
 
 ```python
@@ -226,7 +263,10 @@ Step 2: EXECUTION
 | `min_period_days` | `int` | `30` | Minimum days per sub-period |
 | `min_entities_per_basket` | `int` | `1` | Lower bound on entities per basket |
 | `category` | `CategoryFilter \| dict \| None` | `None` | Optional category filter (`mode` + `values`) |
+| `sentiment` | `SentimentRange \| dict \| list \| None` | `None` | Optional sentiment filter: one inclusive `{min, max}` range or a list of disjoint ranges |
 | `content_diversification` | `bool` | `True` | Enable/disable content diversification in ranking |
+| `requests_per_minute` | `int` | `100` | Global rate limit shared across all planning HTTP work (co-mention + volume) |
+| `max_workers` | `int` | `40` | Maximum concurrent in-flight planning requests (shared semaphore) |
 
 ### `execute_search()`
 
@@ -249,6 +289,7 @@ Step 2: EXECUTION
 - `load_universe_from_csv(csv_path)` -- load entity IDs from a CSV only.
 - `save_plan(plan, path)` / `load_plan(path)` -- persist plans as JSON.
 - `validate_category_filter(category)` -- validate and normalize a category filter to the API dict shape.
+- `validate_sentiment_range(sentiment)` -- validate and normalize sentiment range(s) into the list-of-`{min, max}` dicts emitted as `filters.sentiment.ranges`.
 - `build_comention_payload(...)`, `build_volume_payload(...)`, `build_search_query(...)` -- low-level payload builders used by the planner; useful if you need to construct custom requests.
 
 ### Public Types
@@ -256,6 +297,8 @@ Step 2: EXECUTION
 - `CategoryFilter` -- frozen dataclass with `mode: CategoryMode` and `values: tuple[str, ...]`.
 - `CategoryMode` -- `StrEnum` with `INCLUDE` and `EXCLUDE` values.
 - `CategoryFilterInput` -- type alias: `CategoryFilter | dict[str, str | list[str]]`.
+- `SentimentRange` -- frozen dataclass with inclusive `min: float` and `max: float`.
+- `SentimentRangeInput` -- type alias for one `SentimentRange` / `{min, max}` dict, or a non-empty list of either.
 - `UniverseInput` -- type alias: `str | list[str]`.
 - `VALID_CATEGORY_VALUES` -- `frozenset[str]` of allowed category strings.
 
@@ -331,7 +374,7 @@ Ties are resolved using `VOLUME_BUCKET_PRIORITY`: `high > medium > low > very_lo
 ## Changelog
 
 See [CHANGELOG.md](CHANGELOG.md) for release notes. The package is currently at
-version **1.0.1**.
+version **1.2.0**.
 
 ## License
 

{bigdata_smart_batching-1.1.0 → bigdata_smart_batching-1.2.1}/README.md

@@ -13,6 +13,8 @@ This module provides a two-step pipeline for efficient semantic search:
 - **Parallel execution** -- rate-limited concurrent requests with semaphore-controlled concurrency.
 - **Proportional sampling** -- retrieve a percentage of results while preserving the distribution across baskets.
 - **Category filtering** -- restrict searches to specific document categories (news, transcripts, filings, ...).
+- **Sentiment filtering** -- restrict searches to one or more inclusive document-level sentiment ranges.
+- **Throughput controls** -- shared sliding-window rate limiter and concurrency semaphore across all planning HTTP work, with shared exponential backoff on `429` / `403` / timeout retries.
 - **Deterministic plans** -- every request stays within service limits; plans can be saved and replayed.
 - **Scalable** -- handles universes with 10,000+ companies efficiently.
 
@@ -146,6 +148,41 @@ Valid category values are exposed via `VALID_CATEGORY_VALUES` and include:
 `news_public`, `podcasts`, `research`, `research_academic_journals`,
 `research_investment_research`, `transcripts`.
 
+### Sentiment Filtering
+
+Restrict searches to documents whose sentiment falls inside one or more
+inclusive `min` / `max` ranges using the `sentiment` argument. It accepts a
+single `SentimentRange` / dict, or a list of them (multiple disjoint ranges).
+The filter is emitted as `query.filters.sentiment.ranges` on co-mention,
+volume, and search payloads.
+
+```python
+from bigdata_smart_batching import SentimentRange, plan_search
+
+plan = plan_search(
+    universe="id_name_mapping_us_top_3000.csv",
+    start_date="2023-01-01",
+    end_date="2023-12-31",
+    text="earnings revenue profit",
+    sentiment=SentimentRange(min=0.3, max=1.0),
+)
+
+plan = plan_search(
+    universe="id_name_mapping_us_top_3000.csv",
+    start_date="2023-01-01",
+    end_date="2023-12-31",
+    text="earnings revenue profit",
+    sentiment=[
+        {"min": -1.0, "max": -0.5},
+        {"min": 0.5, "max": 1.0},
+    ],
+)
+```
+
+Each range must satisfy `min <= max`. Use
+`validate_sentiment_range(sentiment)` to validate and normalize input into the
+list-of-dicts shape the API expects.
+
 ### Save and Load Plans
 
 ```python
@@ -210,7 +247,10 @@ Step 2: EXECUTION
 | `min_period_days` | `int` | `30` | Minimum days per sub-period |
 | `min_entities_per_basket` | `int` | `1` | Lower bound on entities per basket |
 | `category` | `CategoryFilter \| dict \| None` | `None` | Optional category filter (`mode` + `values`) |
+| `sentiment` | `SentimentRange \| dict \| list \| None` | `None` | Optional sentiment filter: one inclusive `{min, max}` range or a list of disjoint ranges |
 | `content_diversification` | `bool` | `True` | Enable/disable content diversification in ranking |
+| `requests_per_minute` | `int` | `100` | Global rate limit shared across all planning HTTP work (co-mention + volume) |
+| `max_workers` | `int` | `40` | Maximum concurrent in-flight planning requests (shared semaphore) |
 
 ### `execute_search()`
 
@@ -233,6 +273,7 @@ Step 2: EXECUTION
 - `load_universe_from_csv(csv_path)` -- load entity IDs from a CSV only.
 - `save_plan(plan, path)` / `load_plan(path)` -- persist plans as JSON.
 - `validate_category_filter(category)` -- validate and normalize a category filter to the API dict shape.
+- `validate_sentiment_range(sentiment)` -- validate and normalize sentiment range(s) into the list-of-`{min, max}` dicts emitted as `filters.sentiment.ranges`.
 - `build_comention_payload(...)`, `build_volume_payload(...)`, `build_search_query(...)` -- low-level payload builders used by the planner; useful if you need to construct custom requests.
 
 ### Public Types
@@ -240,6 +281,8 @@ Step 2: EXECUTION
 - `CategoryFilter` -- frozen dataclass with `mode: CategoryMode` and `values: tuple[str, ...]`.
 - `CategoryMode` -- `StrEnum` with `INCLUDE` and `EXCLUDE` values.
 - `CategoryFilterInput` -- type alias: `CategoryFilter | dict[str, str | list[str]]`.
+- `SentimentRange` -- frozen dataclass with inclusive `min: float` and `max: float`.
+- `SentimentRangeInput` -- type alias for one `SentimentRange` / `{min, max}` dict, or a non-empty list of either.
 - `UniverseInput` -- type alias: `str | list[str]`.
 - `VALID_CATEGORY_VALUES` -- `frozenset[str]` of allowed category strings.
 
@@ -315,7 +358,7 @@ Ties are resolved using `VOLUME_BUCKET_PRIORITY`: `high > medium > low > very_lo
 ## Changelog
 
 See [CHANGELOG.md](CHANGELOG.md) for release notes. The package is currently at
-version **1.0.1**.
+version **1.2.0**.
 
 ## License
 

{bigdata_smart_batching-1.1.0 → bigdata_smart_batching-1.2.1}/examples/example.py

@@ -55,21 +55,23 @@ def main() -> None:
             plan = load_plan("plan.json")
         else:
             plan = plan_search(
-                #universe=universe_csv,
-                universe=['54F271'],
+                universe=universe_csv,
+                #universe=['54F271'],
                 start_date="2025-01-01",
-                end_date="2025-12-12",
-                text="Decline in customer confidence in the company",
+                end_date="2025-12-31",
+                text="Company has significant pricing power",
                 volume_query_mode="iterative",
                 apply_volume_splits=True,
-                min_period_days=30,
-                min_entities_per_basket=10,
+                min_period_days=120,
+                min_entities_per_basket=20,
                 # Optional: filter by category to restrict document sources.
                 # Accepts a CategoryFilter instance or a plain dict:
                 category={"mode": "INCLUDE", "values": ["news_premium"]},
                 content_diversification=False,
+                sentiment=[{"min": -1, "max": -0.5},{"min": 0.5, "max": 1}],
             )
             plans.append(plan)
+            save_plan(plan, "plan.json")
             print(f"Chunk upper bound estimate: {plan['chunk_upper_bound_estimate']:,}")
 
             # Export plan
@@ -78,12 +80,12 @@ def main() -> None:
         print(f"Plan execution time: {plan_elapsed_s:.3f}s")
 
         # Step 2: Execute search with 10% of total chunks (preserves distribution)
-        if True:
+        if False:
             search_t0 = time.perf_counter()
             results_raw = execute_search(
                 search_plan=plan,
                 chunk_percentage=1,
-                requests_per_minute=400,
+                requests_per_minute=300,
             )
             search_elapsed_s = time.perf_counter() - search_t0
             print(f"Search execution time: {search_elapsed_s:.3f}s")
@@ -108,10 +110,10 @@ def main() -> None:
         #print(df.head())
         #results_dfs.append(results)
     # Merge dataframe 0 and 1 into a single dataframe using id and chunk index as keys using a full outer join
-    print(plans[0]==plans[1])
+    #print(plans[0]==plans[1])
     #merged = results_dfs[0].merge(results_dfs[1], on=['doc_id', 'chunk_cnum'], how='outer', indicator=True)
     #print(f"% overlapping chunks: {(len(merged[merged._merge!='both'])/2)/len(merged)*100:.2f}%")
-    print('a')
+    #print('a')
 
 
 if __name__ == "__main__":

{bigdata_smart_batching-1.1.0 → bigdata_smart_batching-1.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "bigdata-smart-batching"
-version = "1.1.0"
+version = "1.2.1"
 description = "High-performance semantic search with intelligent company grouping and parallel execution"
 readme = "README.md"
 authors = [{ name = "Bigdata.com", email = "support@ravenpack.com" }]

{bigdata_smart_batching-1.1.0 → bigdata_smart_batching-1.2.1}/src/bigdata_smart_batching/__init__.py

@@ -13,6 +13,8 @@ from .search_function import (
     CategoryFilter,
     CategoryFilterInput,
     CategoryMode,
+    SentimentRange,
+    SentimentRangeInput,
     UniverseInput,
     build_comention_payload,
     build_search_query,
@@ -25,12 +27,15 @@ from .search_function import (
     resolve_universe,
     save_plan,
     validate_category_filter,
+    validate_sentiment_range,
 )
 
 __all__ = [
     "CategoryFilter",
     "CategoryFilterInput",
     "CategoryMode",
+    "SentimentRange",
+    "SentimentRangeInput",
     "UniverseInput",
     "VALID_CATEGORY_VALUES",
     "build_comention_payload",
@@ -45,4 +50,5 @@ __all__ = [
     "resolve_universe",
     "save_plan",
     "validate_category_filter",
+    "validate_sentiment_range",
 ]