equity-aggregator 0.1.1__tar.gz → 0.1.4__tar.gz

{equity_aggregator-0.1.1 → equity_aggregator-0.1.4}/.env_example

@@ -15,7 +15,7 @@ CACHE_TTL_MINUTES=1440
 # Optional database storage location (defaults to system log directory)
 DATA_STORE_DIR=
 
-# authoritative and enrichment feed api keys
+# discovery and enrichment feed api keys
 EXCHANGE_RATE_API_KEY=
 OPENFIGI_API_KEY=
 INTRINIO_API_KEY=

{equity_aggregator-0.1.1 → equity_aggregator-0.1.4}/.github/workflows/publish-build-release.yml

@@ -44,6 +44,11 @@ jobs:
           chmod 600 .env
           test -s .env || { echo "::error ::empty .env"; exit 1; }
 
+      - name: Run live sanity tests
+        env:
+          UV_ENV_FILE: ".env"
+        run: uv run pytest -m live -v
+
       - name: Run Equity Aggregator
         env:
           UV_ENV_FILE: ".env"
@@ -55,6 +60,19 @@ jobs:
           uv run equity-aggregator --verbose seed
           uv run equity-aggregator export --output-dir ./data/data_store
 
+      - name: Get latest log file
+        id: latest_log
+        run: |
+          LATEST_LOG=$(ls -t data/logs/equity_aggregator_*.log | head -1)
+          echo "latest_log=$LATEST_LOG" >> $GITHUB_OUTPUT
+
+      - name: Delete existing release
+        run: |
+          gh release delete latest-build-release --yes || true
+          git push --delete origin latest-build-release || true
+        env:
+          GH_TOKEN: ${{ github.token }}
+
       - name: Publish Latest Canonical Equities Release
         uses: softprops/action-gh-release@v2
         with:
@@ -66,4 +84,4 @@ jobs:
           generate_release_notes: false
           files: |
             data/data_store/canonical_equities.jsonl.gz
-            data/logs/equity_aggregator_*.log
+            ${{ steps.latest_log.outputs.latest_log }}

equity_aggregator-0.1.4/.github/workflows/publish-pypi.yml

@@ -0,0 +1,103 @@
+name: publish-pypi
+
+on:
+  workflow_run:
+    workflows: ["validate-push"]
+    types:
+      - completed
+    branches: [master]
+
+permissions:
+  contents: write
+
+concurrency:
+  group: publish-pypi
+  cancel-in-progress: false
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.conclusion == 'success' &&
+      github.actor != 'github-actions[bot]'
+    outputs:
+      should_publish: ${{ steps.check_commit.outputs.should_publish }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Check if last commit is version bump
+        id: check_commit
+        run: |
+          LAST_COMMIT_MSG=$(git log -1 --pretty=%B)
+          if [[ "$LAST_COMMIT_MSG" == *"chore: bump version"* ]]; then
+            echo "should_publish=false" >> $GITHUB_OUTPUT
+          else
+            echo "should_publish=true" >> $GITHUB_OUTPUT
+          fi
+
+  publish:
+    needs: check
+    if: needs.check.outputs.should_publish == 'true'
+    runs-on: ubuntu-latest
+    env:
+      UV_SYSTEM_PYTHON: 1
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.EQUITY_AGGREGATOR }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install uv (with cache)
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "latest"
+          enable-cache: true
+          cache-dependency-glob: |
+            **/pyproject.toml
+            **/uv.lock
+
+      - name: Sync environment
+        run: uv sync --locked
+
+      - name: Bump patch version
+        id: bump_version
+        run: |
+          CURRENT_VERSION=$(sed -n '3p' pyproject.toml | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')
+          IFS='.' read -r MAJOR MINOR PATCH <<< "$CURRENT_VERSION"
+          NEW_VERSION="${MAJOR}.${MINOR}.$((PATCH + 1))"
+          sed -i "3s/.*/version = \"$NEW_VERSION\"/" pyproject.toml
+          echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Commit and tag version bump
+        run: |
+          git add pyproject.toml
+          git commit -m "chore: bump version to ${{ steps.bump_version.outputs.new_version }} [skip ci]"
+          git tag "v${{ steps.bump_version.outputs.new_version }}"
+
+      - name: Push changes
+        run: |
+          git push origin master
+          git push origin "v${{ steps.bump_version.outputs.new_version }}"
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI }}
+          attestations: false

{equity_aggregator-0.1.1 → equity_aggregator-0.1.4}/.github/workflows/validate-push.yml

@@ -60,7 +60,7 @@ jobs:
         if: success()
         uses: schneegans/dynamic-badges-action@v1.7.0
         with:
-          auth: ${{ secrets.GIST_SECRET }}
+          auth: ${{ secrets.GIST_AUTH }}
           gistID: ce6ce2d7e3c247c34aba66dedcd7ede3
           filename: coverage-badge.json
           label: Coverage

{equity_aggregator-0.1.1 → equity_aggregator-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: equity-aggregator
-Version: 0.1.1
+Version: 0.1.4
 Summary: Financial equity data aggregation toolkit
 Project-URL: Homepage, https://github.com/gregorykelleher/equity-aggregator
 Project-URL: Repository, https://github.com/gregorykelleher/equity-aggregator
@@ -18,13 +18,6 @@ Requires-Dist: openfigipy>=0.1.6
 Requires-Dist: platformdirs>=4.0.0
 Requires-Dist: pydantic>=2.11.3
 Requires-Dist: rapidfuzz>=3.13.0
-Provides-Extra: dev
-Requires-Dist: pytest-asyncio>=1.0.0; extra == 'dev'
-Requires-Dist: pytest-cov>=6.2.0; extra == 'dev'
-Requires-Dist: pytest-env>=1.1.5; extra == 'dev'
-Requires-Dist: pytest-timeout>=2.4.0; extra == 'dev'
-Requires-Dist: pytest>=8.3.5; extra == 'dev'
-Requires-Dist: ruff>=0.8.0; extra == 'dev'
 Description-Content-Type: text/markdown
 
 # Equity Aggregator
@@ -34,20 +27,34 @@ Description-Content-Type: text/markdown
 [![Licence](https://img.shields.io/badge/license-MIT-green)](LICENCE.txt)
 [![Validation Status](https://img.shields.io/github/actions/workflow/status/gregorykelleher/equity-aggregator/validate-push.yml?branch=master&label=build)](https://github.com/gregorykelleher/equity-aggregator/actions/workflows/validate-push.yml)
 [![Coverage](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/gregorykelleher/ce6ce2d7e3c247c34aba66dedcd7ede3/raw/coverage-badge.json)](https://github.com/gregorykelleher/equity-aggregator/actions/workflows/validate-push.yml)
-[![CodeFactor](https://www.codefactor.io/repository/github/gregorykelleher/equity-aggregator/badge/master)](https://www.codefactor.io/repository/github/gregorykelleher/equity-aggregator/overview/master)
 
 ## Description
 
-Equity Aggregator is a financial data tool that collects and normalises raw equity data from authoritative sources (Euronext, LSE, SEC, XETRA), before enriching it with third-party market vendor data to produce a unified canonical dataset of unique equities.
+Equity Aggregator is a financial data tool that collects and normalises raw equity data from discovery sources (Intrinio, LSEG, SEC, XETRA, Stock Analysis, TradingView), before enriching it with third-party market vendor data from enrichment feeds (Yahoo Finance and Global LEI Foundation) to produce a unified canonical dataset of unique equities.
 
-Altogether, this tool makes it possible to retrieve up-to-date information on over 9,500+ equities from ten countries worldwide:
+Altogether, this tool makes it possible to retrieve up-to-date information on over 15,000+ equities from countries worldwide.
 
-| Source | Country | Description |
+### Discovery Feeds
+
+Discovery feeds provide raw equity data from primary market sources:
+
+| Source | Coverage | Description |
 |----------|---------|-------------|
-| 🇪🇺 Euronext | Europe | Pan-European stock exchange operating in Netherlands, France, Belgium, Portugal, Ireland, and Norway |
-| 🇬🇧 LSE | United Kingdom | London Stock Exchange |
-| 🇺🇸 SEC | United States | Securities and Exchange Commission |
-| 🇩🇪 XETRA | Germany | Deutsche Börse electronic trading platform |
+| 🇺🇸 Intrinio | United States | Intrinio - US-listed equities |
+| 🇬🇧 LSEG | International | London Stock Exchange Group - Global equities |
+| 🇺🇸 SEC | United States | Securities and Exchange Commission - US-listed equities |
+| 🇺🇸 Stock Analysis | United States | Stock Analysis - Global equities |
+| 🇺🇸 TradingView | United States | TradingView - US-listed equities |
+| 🇩🇪 XETRA | Germany | Deutsche Börse electronic trading platform - Global listed equities |
+
+### Enrichment Feeds
+
+Enrichment feeds provide supplementary data to enhance the canonical equity dataset:
+
+| Source | Description |
+|--------|-------------|
+| Yahoo Finance | Market data, financial metrics, and equity metadata |
+| GLEIF | Legal Entity Identifier (LEI) lookups via the Global LEI Foundation |
 
 ## What kind of Equity Data is available?
 
@@ -58,10 +65,11 @@ Equity Aggregator provides a comprehensive profile for each equity in its canoni
 |-------|-------------|
 | **name** | Full company name |
 | **symbol** | Trading symbol |
-| **share class figi** | Authoritative OpenFIGI identifier |
+| **share class figi** | Definitive OpenFIGI identifier |
 | **isin** | International Securities Identification Number |
 | **cusip** | CUSIP identifier |
 | **cik** | Central Index Key for SEC filings |
+| **lei** | Legal Entity Identifier (ISO 17442) |
 
 ### Financial Metrics
 | Category | Fields |
@@ -79,7 +87,7 @@ Equity Aggregator provides a comprehensive profile for each equity in its canoni
 | **Classification** | `industry`, `sector`, `analyst_rating` |
 
 > [!NOTE]
-> The OpenFIGI Share Class FIGI is the only definitive unique identifier for each equity in this dataset. While other identifiers like ISIN, CUSIP, and CIK are also collected, they may not be universally available across all global markets or may have inconsistencies in formatting and coverage.
+> The OpenFIGI Share Class FIGI is the only definitive unique identifier for each equity in this dataset. While other identifiers like ISIN, CUSIP, CIK and LEI are also collected, they may not be universally available across all global markets or may have inconsistencies in formatting and coverage.
 >
 > OpenFIGI provides standardised, globally unique identifiers that work consistently across all equity markets and exchanges, hence its selection for Equity Aggregator.
 
@@ -93,91 +101,9 @@ Equity Aggregator is available to download via `pip` as the `equity-aggregator`
 pip install equity-aggregator
 ```
 
-### CLI Usage
-
-Once installed, Equity Aggregator provides a comprehensive command-line interface for managing equity data operations. The CLI offers three main commands:
-
-- **seed** - Aggregate and populate the local database with fresh equity data
-- **export** - Export the local canonical equity database to compressed JSONL format
-- **download** - Download the latest canonical equity data from remote repository
-
-Run `equity-aggregator --help` for more information:
-
-```bash
-usage: equity-aggregator [-h] [-v] [-d] [-q] {seed,export,download} ...
-
-aggregate, download, and export canonical equity data
-
-options:
-  -h, --help            show this help message and exit
-  -v, --verbose         enable verbose logging (INFO level)
-  -d, --debug           enable debug logging (DEBUG level)
-  -q, --quiet           quiet mode - only show warnings and errors
-
-commands:
-  Available operations
-
-  {seed,export,download}
-    seed                aggregate enriched canonical equity data sourced from data feeds
-    export              export local canonical equity data to compressed JSONL format
-    download            download latest canonical equity data from remote repository
-
-Use 'equity-aggregator <command> --help' for help
-```
-
-#### Download Command
-
-The `download` command retrieves the latest pre-processed canonical equity dataset from GitHub Releases, eliminating the need to run the full aggregation pipeline via `seed` locally. This command:
-
-- Downloads compressed equity data (`canonical_equities.jsonl.gz`) from the latest nightly build
-- Automatically rebuilds the database locally from the downloaded data
-- Provides access to 9,500+ equities with immediate effect
-
-> [!TIP]
-> **Optional: Increase Rate Limits**
->
-> Set `GITHUB_TOKEN` to increase download limits from 60/hour to 5,000/hour:
-> ```bash
-> export GITHUB_TOKEN="your_personal_access_token_here"
-> ```
-> Create a token at [GitHub Settings](https://github.com/settings/tokens) - no special scopes needed. Recommended for frequent downloads or CI/CD pipelines.
-
-#### Export Command
-
-The `export` command extracts canonical equity data from the local database and exports it as compressed JSONL (JSON Lines) format. It reads all canonical equities from the local database and exports the data to `canonical_equities.jsonl.gz` in the specified output directory.
-
-This creates a portable, standardised dataset suitable for analysis, sharing, or backup while preserving all equity metadata and financial metrics in structured JSON format.
-
-```bash
-# Export aggregated data to compressed JSON in specified directory
-equity-aggregator export --output-dir ~/Downloads
-equity-aggregator export --output-dir /path/to/export/location
-```
-
-#### Seed Command
-
-The `seed` command executes the complete equity aggregation pipeline, collecting raw data from authoritative sources (Euronext, LSE, SEC, XETRA), enriching it with market data from enrichment feeds, and storing the processed results in the local database. This command runs the full transformation pipeline to create a fresh canonical equity dataset.
-
-This command requires that the following API keys are set prior:
-
-```bash
-export EXCHANGE_RATE_API_KEY="your_key_here"
-export OPENFIGI_API_KEY="your_key_here"
-```
-
-```bash
-# Run the main aggregation pipeline (requires API keys)
-equity-aggregator seed
-```
+### Python API
 
-> [!IMPORTANT]
-> Note that the `seed` command processes thousands of equities and is intentionally rate-limited to respect external API constraints. A full run typically takes 90 minutes depending on network conditions and API response times.
->
-> This is mitigated by the automated nightly CI pipeline that runs `seed` and publishes the latest canonical equity dataset. Users can download this pre-built data using `equity-aggregator download` instead of running the full aggregation pipeline locally.
-
-### Python API Integration
-
-Beyond the CLI, Equity Aggregator also exposes a focused public API that enables seamless integration opportunities. The API automatically detects and downloads the latest canonical equity dataset from remote sources when needed, ensuring users always work with up-to-date data.
+Equity Aggregator exposes a focused public API that enables seamless integration opportunities. The API automatically detects and downloads the latest canonical equity dataset from remote sources when needed, ensuring users always work with up-to-date data.
 
 #### Retrieving All Equities
 
@@ -197,7 +123,7 @@ for equity in equities[:3]:  # Show first 3
 
 **Example Output:**
 ```
-Retrieved 9547 canonical equities
+Retrieved 10000 canonical equities
 AAPL: APPLE INC
 MSFT: MICROSOFT CORP
 GOOGL: ALPHABET INC
@@ -260,6 +186,88 @@ Market Cap: 3500000000000
 > [!NOTE]
 > Both functions work independently - `retrieve_canonical_equity()` automatically downloads data if needed, so there's no requirement to call `retrieve_canonical_equities()` first.
 
+### CLI Usage
+
+Once installed, Equity Aggregator provides a comprehensive command-line interface for managing equity data operations. The CLI offers three main commands:
+
+- **seed** - Aggregate and populate the local database with fresh equity data
+- **export** - Export the local canonical equity database to compressed JSONL format
+- **download** - Download the latest canonical equity data from remote repository
+
+Run `equity-aggregator --help` for more information:
+
+```bash
+usage: equity-aggregator [-h] [-v] [-d] [-q] {seed,export,download} ...
+
+aggregate, download, and export canonical equity data
+
+options:
+  -h, --help            show this help message and exit
+  -v, --verbose         enable verbose logging (INFO level)
+  -d, --debug           enable debug logging (DEBUG level)
+  -q, --quiet           quiet mode - only show warnings and errors
+
+commands:
+  Available operations
+
+  {seed,export,download}
+    seed                aggregate enriched canonical equity data sourced from data feeds
+    export              export local canonical equity data to compressed JSONL format
+    download            download latest canonical equity data from remote repository
+
+Use 'equity-aggregator <command> --help' for help
+```
+
+#### Download Command
+
+The `download` command retrieves the latest pre-processed canonical equity dataset from GitHub Releases, eliminating the need to run the full aggregation pipeline via `seed` locally. This command:
+
+- Downloads compressed equity data (`canonical_equities.jsonl.gz`) from the latest nightly build
+- Automatically rebuilds the database locally from the downloaded data
+- Provides access to 15,000+ equities with immediate effect
+
+> [!TIP]
+> **Optional: Increase Rate Limits**
+>
+> Set `GITHUB_TOKEN` to increase download limits from 60/hour to 5,000/hour:
+> ```bash
+> export GITHUB_TOKEN="your_personal_access_token_here"
+> ```
+> Create a token at [GitHub Settings](https://github.com/settings/tokens) - no special scopes needed. Recommended for frequent downloads or CI/CD pipelines.
+
+#### Export Command
+
+The `export` command extracts canonical equity data from the local database and exports it as compressed JSONL (JSON Lines) format. It reads all canonical equities from the local database and exports the data to `canonical_equities.jsonl.gz` in the specified output directory.
+
+This creates a portable, standardised dataset suitable for analysis, sharing, or backup while preserving all equity metadata and financial metrics in structured JSON format.
+
+```bash
+# Export aggregated data to compressed JSON in specified directory
+equity-aggregator export --output-dir ~/Downloads
+equity-aggregator export --output-dir /path/to/export/location
+```
+
+#### Seed Command
+
+The `seed` command executes the complete equity aggregation pipeline, collecting raw data from discovery sources (LSEG, SEC, XETRA, Stock Analysis, TradingView), enriching it with market data from enrichment feeds, and storing the processed results in the local database. This command runs the full transformation pipeline to create a fresh canonical equity dataset.
+
+This command requires that the following API keys are set prior:
+
+```bash
+export EXCHANGE_RATE_API_KEY="your_key_here"
+export OPENFIGI_API_KEY="your_key_here"
+```
+
+```bash
+# Run the main aggregation pipeline (requires API keys)
+equity-aggregator seed
+```
+
+> [!IMPORTANT]
+> Note that the `seed` command processes thousands of equities and is intentionally rate-limited to respect external API constraints. A full run typically takes 60 minutes depending on network conditions and API response times.
+>
+> This is mitigated by the automated nightly CI pipeline that runs `seed` and publishes the latest canonical equity dataset. Users can download this pre-built data using `equity-aggregator download` instead of running the full aggregation pipeline locally.
+
 ### Data Storage
 
 Equity Aggregator automatically stores its database (i.e. `data_store.db`) in system-appropriate locations using platform-specific directories:
@@ -339,9 +347,9 @@ cp .env_example .env
 
 #### Optional Keys:
 
-- `INTRINIO_API_KEY` - For additional data enrichment
+- `INTRINIO_API_KEY` - For Intrinio discovery feed
   - Retrieve from: [Intrinio](https://intrinio.com/)
-  - Provides supplementary equity enrichment data
+  - Provides US equity data with comprehensive quote information
 
 - `GITHUB_TOKEN` - For increased GitHub API rate limits
   - Retrieve from: [GitHub Settings](https://github.com/settings/tokens)
@@ -440,7 +448,7 @@ docker compose exec equity-aggregator bash
 ```
 
 > [!NOTE]
-> The Docker container mounts the `data/` directory as a volume for persistent database storage.
+> The Docker setup uses named volumes for persistent database storage and automatically handles all directory creation and permissions.
 
 ## Architecture
 
@@ -455,8 +463,8 @@ equity-aggregator/
 │   ├── domain/pipeline/             # Core aggregation pipeline
 │   │   └── transforms/              # Transformation stages
 │   ├── adapters/data_sources/       # External data integrations
-│   │   ├── authoritative_feeds/     # Primary sources (Euronext, LSE, SEC, XETRA)
-│   │   └── enrichment_feeds/        # Yahoo Finance integration
+│   │   ├── discovery_feeds/         # Primary sources (Intrinio, LSEG, SEC, Stock Analysis, TradingView, XETRA)
+│   │   └── enrichment_feeds/        # Enrichment feed integrations (Yahoo Finance)
 │   ├── schemas/                     # Data validation and types
 │   └── storage/                     # Database operations
 ├── data/                            # Database and cache
@@ -465,15 +473,29 @@ equity-aggregator/
 └── pyproject.toml                   # Project metadata and dependencies
 ```
 
+### Project Dependencies (Production)
+
+The dependency listing is intentionally minimal, relying only on the following core packages:
+
+| Dependency | Use case |
+|------------|----------|
+| pydantic | Type-safe models and validation for data |
+| rapidfuzz | Fast fuzzy matching to reconcile data sourced by multiple data feeds |
+| httpx | HTTP client with HTTP/2 support for data feed retrieval |
+| openfigipy | OpenFIGI integration that anchors equities to a definitive identifier |
+| platformdirs | Consistent storage paths for caches, logs, and data stores on every OS |
+
+Keeping such a small set of dependencies reduces upgrade risk and maintenance costs, whilst still providing all the functionality required for comprehensive equity data aggregation and processing.
+
 ### Data Transformation Pipeline
 
 The aggregation pipeline consists of six sequential transformation stages, each with a specific responsibility:
 
-1. **Parse**: Extract and validate raw equity data from authorative feed data
+1. **Parse**: Extract and validate raw equity data from discovery feed data
 2. **Convert**: Normalise currency values to USD reference currency using live exchange rates
-3. **Identify**: Attach authoritative identification metadata (i.e. Share Class FIGI) via OpenFIGI API integration
-4. **Deduplicate**: Merge duplicate equity records predicated on Share Class FIGI
-5. **Enrich**: Supplement core data with additional market metrics sourced from enrichment feeds
+3. **Identify**: Attach definitive identification metadata (i.e. Share Class FIGI) via OpenFIGI
+4. **Group**: Group equities by Share Class FIGI, preserving all discovery feed sources
+5. **Enrich**: Fetch enrichment data and perform single comprehensive merge of all sources (discovery + enrichment)
 6. **Canonicalise**: Transform enriched data into the final canonical equity schema
 
 ### Clean Architecture Layers
@@ -485,16 +507,86 @@ The codebase adheres to clean architecture principles with distinct layers:
 - **Infrastructure Layer** (`storage/`, `cli/`): Handles system concerns, regarding database operations and command-line tooling
 - **Schema Layer** (`schemas/`): Defines data contracts and validation rules using Pydantic models for type safety
 
+### Test Suites
+
+The project maintains two distinct test suites, each serving a specific purpose in the testing strategy:
+
+#### Unit Tests (`-m unit`)
+
+Unit tests provide comprehensive coverage of all internal application logic. These tests are fully isolated and do not make any external network calls, ensuring fast and deterministic execution. The suite contains over 1,000 test cases and executes in under 30 seconds, enforcing a **minimum coverage threshold of 99%** with the goal of maintaining **100% coverage** across all source code.
+
+Unit tests follow strict conventions:
+- **AAA Pattern**: All tests are structured using the Arrange-Act-Assert pattern for clarity and consistency
+- **Single Assertion**: Each test case contains exactly one assertion, ensuring focused and maintainable tests
+- **No Mocking**: Monkey-patching and Python mocking techniques (e.g. `monkeypatch`, `unittest.mock`) are strictly forbidden, promoting testable design through dependency injection and explicit interfaces
+
+#### Live Tests (`-m live`)
+
+Live tests serve as **sanity tests** that validate external API endpoints are available and responding correctly. These tests hit real external services to verify that:
+- Discovery and enrichment feed endpoints are accessible
+- API response schemas match expected Pydantic models
+- Authentication and rate limiting are functioning as expected
+
+Live tests act as an early warning system, catching upstream API changes or outages before they impact the main aggregation pipeline.
+
+#### Continuous Integration
+
+Both test suites are executed as part of the GitHub Actions CI pipeline:
+
+- **[validate-push.yml](.github/workflows/validate-push.yml)**: Runs unit tests with coverage enforcement on every push to master, ensuring code quality and the 99% coverage threshold are maintained
+- **[publish-build-release.yml](.github/workflows/publish-build-release.yml)**: Runs live sanity tests before executing the nightly aggregation pipeline, validating that all external APIs are operational before publishing a new release
+
+## Limitations
+
+### Data Depth and Scope
+
+- Equity Aggregator is intrinsically bound by the quality and coverage of its upstream discovery and enrichment feeds. Data retrieved and processed by Equity Aggregator reflects the quality and scope inherited from these data sources.
+
+- Normalisation, outlier detection, coherency validation checks and other statistical techniques catch most upstream issues, yet occasional gaps or data aberrations can persist and should be handled defensively by downstream consumers.
+
+### Venue-Specific Financial Metrics and Secondary Listings
+
+- Certain equities may be sourced solely from secondary listings (e.g. OTC Markets or cross-listings) rather than their primary exchange. This occurs when the primary venue's data is unavailable from equity-aggregator's data sources.
+
+- Company-level metrics such as `market_cap`, `shares_outstanding`, `revenue`, and valuation ratios remain accurate regardless of sourcing venue, as they reflect the underlying company rather than the trading venue.
+
+- However, venue-specific metrics, particularly `market_volume` reflect trading activity only on the captured venues, not _total_ market-wide volume. An equity showing low volume may simply indicate minimal OTC activity despite substantial trading on its primary exchange.
+
+- Attention should therefore be paid to the  `mics` field, indicating which Market Identifier Codes are represented in the data (i.e. whether it's the equity's primary exchange MIC or a secondary listing).
+
+### Data Update Cadence
+
+- Equity Aggregator publishes nightly batch snapshots and does not aim to serve as a real-time market data service. The primary objective of Equity Aggregator is to provide equity identification metadata with limited financial metrics for fundamental analysis.
+
+- Downstream services should therefore treat Equity Aggregator as a discovery catalogue, using its authoritative identifiers to discover equities and then poll specialised market data providers for time-sensitive pricing metrics.
+
+- Delivering real-time quotes directly through Equity Aggregator would be infeasible because the upstream data sources enforce strict rate limits and the pipeline is network-bound; attempting live polling would exhaust quotas quickly and degrade reliability for all consumers.
+
+### Single Identifier Authority
+
+- Share Class FIGI remains the authoritative identifier because OpenFIGI supplies globally unique, deduplicated mappings across discovery feeds. Other identifiers such as ISIN, CUSIP, CIK or LEI depend on regional registries, are frequently absent for specific markets, and are prone to formatting discrepancies, so they should be treated as supplementary identifiers only.
+
+### Performance
+
+- The end-to-end aggregation pipeline is network-bound and respects vendor rate limits, meaning a full `seed` run can take close to an hour in steady-state conditions. This is mitigated by comprehensive caching used throughout the application, as well as the automated nightly CI pipeline that publishes the latest canonical equity dataset, made available via `download`.
+
+### External Service Reliance
+
+- As the entirety of Equity Aggregator is built around the use of third-party APIs for discovery, enrichment, as well as other services, its robustness is fundamentally fragile. Upstream outages, schema shifts, bot protection revocations, API churn and rate-limit policy changes can easily degrade the pipeline without warning, with remediation often relying on vendor response times outside of the project's remit.
+
+- As this is an inherent architectural constraint, the only viable response centres on providing robust mitigation controls. Monitoring, retry strategies and graceful degradation paths lessen the impact; they cannot eliminate the dependency risk entirely.
+
 ## Disclaimer
 
 > [!IMPORTANT]
 > **Important Legal Notice**
 >
-> This software aggregates data from various third-party sources including Yahoo Finance, Euronext, London Stock Exchange, SEC, and XETRA. Equity Aggregator is **not** affiliated, endorsed, or vetted by any of these organisations.
+> This software aggregates data from various third-party sources including Intrinio, Yahoo Finance, LSEG trading platform, SEC, Stock Analysis, and XETRA. Equity Aggregator is **not** affiliated, endorsed, or vetted by any of these organisations.
 >
 > **Data Sources and Terms:**
 >
 > - **Yahoo Finance**: This tool uses Yahoo's publicly available APIs. Refer to [Yahoo!'s terms of use](https://policies.yahoo.com/us/en/yahoo/terms/product-atos/apiforydn/index.htm) for details on your rights to use the actual data downloaded. Yahoo! finance API is intended for personal use only.
+> - **Intrinio**: This tool requires a valid Intrinio subscription and API key. Refer to [Intrinio's terms of use](https://about.intrinio.com/terms) for permitted usage, rate limits, and redistribution policies.
 > - **Market Data**: All market data is obtained from publicly available sources and is intended for research and educational purposes only.
 >
 > **Usage Responsibility:**
@@ -505,5 +597,3 @@ The codebase adheres to clean architecture principles with distinct layers:
 > - Users should independently verify any data before making financial decisions
 >
 > **Commercial Use:** Users intending commercial use should review and comply with the terms of service of all underlying data providers.
-
-