supadata 1.0.0__tar.gz

supadata-1.0.0/.gitignore

@@ -0,0 +1,60 @@
+# Python
+**/__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+**/build/
+develop-eggs/
+**/dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+**/*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual Environment
+.env
+**/.venv/
+**/env/
+**/venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDE
+**/.idea/
+**/.vscode/
+*.swp
+*.swo
+.DS_Store
+
+# Testing
+.coverage
+**/.pytest_cache/
+**/htmlcov/
+**/.tox/
+**/.nox/
+coverage.xml
+*.cover
+*.py,cover
+**/.hypothesis/
+
+# Distribution / packaging
+.Python
+*.manifest
+*.spec
+
+# Jupyter Notebook
+**/.ipynb_checkpoints
+
+# Logs
+*.log 

supadata-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Supadata
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

supadata-1.0.0/PKG-INFO

@@ -0,0 +1,97 @@
+Metadata-Version: 2.4
+Name: supadata
+Version: 1.0.0
+Summary: The official Python SDK for Supadata - scrape web content and YouTube transcripts with ease
+Project-URL: homepage, https://supadata.ai
+Project-URL: repository, https://github.com/supadata/supadata-py
+Project-URL: documentation, https://supadata.ai/documentation
+Author-email: Supadata <support@supadata.ai>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,api,llm,supadata,transcripts,web-scraping,youtube
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Requires-Dist: requests>=2.28.1
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == 'test'
+Requires-Dist: requests-mock>=1.11.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# Supadata Python SDK
+
+[![PyPI version](https://badge.fury.io/py/supadata.svg)](https://badge.fury.io/py/supadata)
+[![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](http://opensource.org/licenses/MIT)
+
+The official Python SDK for Supadata.
+
+Get your free API key at [supadata.ai](https://supadata.ai) and start scraping data in minutes.
+
+## Installation
+
+```bash
+pip install supadata
+```
+
+## Usage
+
+```python
+from supadata import Supadata
+
+# Initialize the client
+client = Supadata(api_key="YOUR_API_KEY")
+
+# Get YouTube transcript
+transcript = client.get_transcript(video_id="VIDEO_ID")
+print(f"Got transcript in {transcript['lang']}")
+
+# Translate YouTube transcript to Spanish
+translated = client.translate_transcript(
+    video_id="VIDEO_ID",
+    lang="es"
+)
+print(f"Got translated transcript in {translated['lang']}")
+
+# Get plain text transcript
+text_transcript = client.get_transcript(
+    video_id="VIDEO_ID",
+    text=True
+)
+print(text_transcript['content'])
+
+# Scrape web content
+web_content = client.scrape("https://supadata.ai")
+print(f"Page title: {web_content['name']}")
+print(f"Content length: {web_content['countCharacters']} characters")
+
+# Map website URLs
+site_map = client.map("https://supadata.ai")
+print(f"Found {len(site_map['urls'])} URLs")
+```
+
+## Error Handling
+
+The SDK uses the standard `requests` library and will raise `requests.exceptions.RequestException` for API-related errors:
+
+```python
+from requests.exceptions import RequestException
+
+try:
+    transcript = client.get_transcript(video_id="INVALID_ID")
+except RequestException as error:
+    print(f"API request failed: {error}")
+    if error.response is not None:
+        error_data = error.response.json()
+        print(f"Error code: {error_data.get('code')}")
+        print(f"Error title: {error_data.get('title')}")
+        print(f"Error description: {error_data.get('description')}")
+```
+
+## API Reference
+
+See the [Documentation](https://supadata.ai/documentation) for more details on all possible parameters and options.
+
+## License
+
+MIT

supadata-1.0.0/README.md

@@ -0,0 +1,76 @@
+# Supadata Python SDK
+
+[![PyPI version](https://badge.fury.io/py/supadata.svg)](https://badge.fury.io/py/supadata)
+[![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](http://opensource.org/licenses/MIT)
+
+The official Python SDK for Supadata.
+
+Get your free API key at [supadata.ai](https://supadata.ai) and start scraping data in minutes.
+
+## Installation
+
+```bash
+pip install supadata
+```
+
+## Usage
+
+```python
+from supadata import Supadata
+
+# Initialize the client
+client = Supadata(api_key="YOUR_API_KEY")
+
+# Get YouTube transcript
+transcript = client.get_transcript(video_id="VIDEO_ID")
+print(f"Got transcript in {transcript['lang']}")
+
+# Translate YouTube transcript to Spanish
+translated = client.translate_transcript(
+    video_id="VIDEO_ID",
+    lang="es"
+)
+print(f"Got translated transcript in {translated['lang']}")
+
+# Get plain text transcript
+text_transcript = client.get_transcript(
+    video_id="VIDEO_ID",
+    text=True
+)
+print(text_transcript['content'])
+
+# Scrape web content
+web_content = client.scrape("https://supadata.ai")
+print(f"Page title: {web_content['name']}")
+print(f"Content length: {web_content['countCharacters']} characters")
+
+# Map website URLs
+site_map = client.map("https://supadata.ai")
+print(f"Found {len(site_map['urls'])} URLs")
+```
+
+## Error Handling
+
+The SDK uses the standard `requests` library and will raise `requests.exceptions.RequestException` for API-related errors:
+
+```python
+from requests.exceptions import RequestException
+
+try:
+    transcript = client.get_transcript(video_id="INVALID_ID")
+except RequestException as error:
+    print(f"API request failed: {error}")
+    if error.response is not None:
+        error_data = error.response.json()
+        print(f"Error code: {error_data.get('code')}")
+        print(f"Error title: {error_data.get('title')}")
+        print(f"Error description: {error_data.get('description')}")
+```
+
+## API Reference
+
+See the [Documentation](https://supadata.ai/documentation) for more details on all possible parameters and options.
+
+## License
+
+MIT

supadata-1.0.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "supadata"
+version = "1.0.0"
+authors = [{ name = "Supadata", email = "support@supadata.ai" }]
+dependencies = ["requests >= 2.28.1"]
+description = "The official Python SDK for Supadata - scrape web content and YouTube transcripts with ease"
+readme = "README.md"
+requires-python = ">=3.7"
+license = "MIT"
+keywords = ["supadata", "web-scraping", "youtube", "transcripts", "api", "llm", "ai"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+homepage = "https://supadata.ai"
+repository = "https://github.com/supadata/supadata-py"
+documentation = "https://supadata.ai/documentation"
+
+[project.optional-dependencies]
+test = [
+    "pytest >= 7.0.0",
+    "requests-mock >= 1.11.0",
+]

supadata-1.0.0/supadata/__init__.py

@@ -0,0 +1,26 @@
+"""
+Supadata Python SDK
+
+The official Python SDK for Supadata - scrape web and YouTube content with ease.
+"""
+from importlib.metadata import version
+from supadata.client import Supadata
+from supadata.types import (
+    Transcript,
+    TranslatedTranscript,
+    TranscriptChunk,
+    Scrape,
+    Map,
+    Error,
+)
+
+__version__ = version("supadata")
+__all__ = [
+    "Supadata",
+    "Transcript",
+    "TranslatedTranscript",
+    "TranscriptChunk",
+    "Scrape",
+    "Map",
+    "Error",
+] 

supadata-1.0.0/supadata/client.py

@@ -0,0 +1,164 @@
+"""Main Supadata client implementation."""
+
+from typing import Dict, Any
+import requests
+from dataclasses import asdict
+
+from .types import (
+    Transcript,
+    TranslatedTranscript,
+    TranscriptChunk,
+    Scrape,
+    Map,
+    Error,
+)
+
+
+class Supadata:
+    """Main Supadata client."""
+
+    def __init__(self, api_key: str, base_url: str = "https://api.supadata.ai/v1"):
+        """Initialize Supadata client.
+
+        Args:
+            api_key: Your Supadata API key
+            base_url: Optional custom API base URL
+        """
+        self.base_url = base_url
+        self.session = requests.Session()
+        self.session.headers.update({
+            "x-api-key": api_key,
+            "Accept": "application/json"
+        })
+
+    def get_transcript(self, video_id: str, text: bool = False) -> Transcript:
+        """Get transcript for a YouTube video.
+
+        Args:
+            video_id: YouTube video ID
+            text: Whether to return plain text instead of segments
+
+        Returns:
+            Transcript object containing content, language and available languages
+
+        Raises:
+            requests.exceptions.RequestException: If the API request fails
+        """
+        response = self._request("GET", "/youtube/transcript", params={
+            "videoId": video_id,
+            "text": text
+        })
+
+        # Convert chunks if present
+        if not text and isinstance(response["content"], list):
+            response["content"] = [
+                TranscriptChunk(**chunk) for chunk in response["content"]
+            ]
+
+        return Transcript(**response)
+
+    def translate_transcript(
+        self,
+        video_id: str,
+        lang: str,
+        text: bool = False
+    ) -> TranslatedTranscript:
+        """Get translated transcript for a YouTube video.
+
+        Args:
+            video_id: YouTube video ID
+            lang: Target language code (e.g., 'es' for Spanish)
+            text: Whether to return plain text instead of segments
+
+        Returns:
+            TranslatedTranscript object containing translated content
+
+        Raises:
+            requests.exceptions.RequestException: If the API request fails
+        """
+        response = self._request("GET", "/youtube/transcript/translate", params={
+            "videoId": video_id,
+            "lang": lang,
+            "text": text
+        })
+
+        # Convert chunks if present
+        if not text and isinstance(response["content"], list):
+            response["content"] = [
+                TranscriptChunk(**chunk) for chunk in response["content"]
+            ]
+
+        return TranslatedTranscript(**response)
+
+    def scrape(self, url: str) -> Scrape:
+        """Scrape content from a web page.
+
+        Args:
+            url: URL to scrape
+
+        Returns:
+            Scrape object containing the extracted content
+
+        Raises:
+            requests.exceptions.RequestException: If the API request fails
+        """
+        response = self._request("GET", "/web/scrape", params={"url": url})
+        return Scrape(**response)
+
+    def map(self, url: str) -> Map:
+        """Generate a site map for a website.
+
+        Args:
+            url: Base URL to map
+
+        Returns:
+            Map object containing discovered URLs
+
+        Raises:
+            requests.exceptions.RequestException: If the API request fails
+        """
+        response = self._request("GET", "/web/map", params={"url": url})
+        return Map(**response)
+
+    def _camel_to_snake(self, d: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert dictionary keys from camelCase to snake_case."""
+        import re
+        def convert(name: str) -> str:
+            name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+            return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower()
+        
+        if isinstance(d, dict):
+            return {convert(k): self._camel_to_snake(v) for k, v in d.items()}
+        if isinstance(d, list):
+            return [self._camel_to_snake(i) for i in d]
+        return d
+
+    def _request(self, method: str, path: str, **kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        """Make an HTTP request to the Supadata API.
+
+        Args:
+            method: HTTP method
+            path: API endpoint path
+            **kwargs: Additional arguments to pass to requests
+
+        Returns:
+            dict: Parsed JSON response
+
+        Raises:
+            requests.exceptions.RequestException: If the API request fails
+        """
+        url = f"{self.base_url}{path}"
+        response = self.session.request(method, url, **kwargs)
+
+        try:
+            response.raise_for_status()
+            return self._camel_to_snake(response.json())
+        except requests.exceptions.HTTPError as e:
+            if e.response is not None:
+                try:
+                    error_data = self._camel_to_snake(e.response.json())
+                    error = Error(**error_data)
+                    raise requests.exceptions.HTTPError(error) from e
+                except (ValueError, TypeError):
+                    pass
+            raise 

supadata-1.0.0/supadata/errors.py

@@ -0,0 +1,31 @@
+"""Custom exceptions for Supadata SDK."""
+
+from typing import Optional
+from dataclasses import dataclass
+
+
+@dataclass
+class SupadataError(Exception):
+    """Base exception for all Supadata errors.
+    
+    Attributes:
+        code: Error code identifying the type of error (e.g., 'video-not-found')
+        title: Human readable error title
+        description: Detailed error description
+        documentation_url: URL to error documentation
+    """
+    code: str
+    title: str
+    description: str
+    documentation_url: Optional[str] = None
+
+    def __str__(self) -> str:
+        """Return string representation of the error."""
+        parts = [self.description]
+        if self.code:
+            parts.append(f"Code: {self.code}")
+        if self.title:
+            parts.append(f"Title: {self.title}")
+        if self.documentation_url:
+            parts.append(f"Documentation: {self.documentation_url}")
+        return " | ".join(parts) 

supadata-1.0.0/supadata/types.py

@@ -0,0 +1,95 @@
+"""Type definitions for Supadata API responses."""
+
+from typing import List, Optional, Union
+from dataclasses import dataclass
+
+
+@dataclass
+class TranscriptChunk:
+    """A chunk of a video transcript.
+    
+    Attributes:
+        text: Transcript segment text
+        offset: Start time in milliseconds
+        duration: Duration in milliseconds
+        lang: ISO 639-1 language code of chunk
+    """
+    text: str
+    offset: int
+    duration: int
+    lang: str
+
+
+@dataclass
+class Transcript:
+    """A complete video transcript.
+    
+    Attributes:
+        content: List of transcript chunks or plain text when text=true
+        lang: ISO 639-1 language code of transcript
+        available_langs: List of available language codes
+    """
+    content: Union[List[TranscriptChunk], str]
+    lang: str
+    available_langs: List[str]
+
+
+@dataclass
+class TranslatedTranscript:
+    """A translated video transcript.
+    
+    Attributes:
+        content: List of transcript chunks or plain text when text=true
+        lang: ISO 639-1 language code of translation
+    """
+    content: Union[List[TranscriptChunk], str]
+    lang: str
+
+
+@dataclass
+class Scrape:
+    """Scraped web content.
+    
+    Attributes:
+        url: The URL that was scraped
+        content: The Markdown content extracted from the URL
+        name: The name of the webpage
+        description: A description of the webpage
+        og_url: Open Graph URL for the webpage
+        count_characters: The number of characters in the content
+        urls: List of URLs found on the webpage
+    """
+    url: str
+    content: str
+    name: str
+    description: str
+    og_url: Optional[str]
+    count_characters: int
+    urls: List[str]
+
+
+@dataclass
+class Map:
+    """A site map containing URLs.
+    
+    Attributes:
+        urls: List of URLs found on the webpage
+    """
+    urls: List[str]
+
+
+@dataclass
+class Error:
+    """Standard error response format.
+    
+    Attributes:
+        code: Error code identifying the type of error
+        title: Human readable error title
+        description: Detailed error description
+        documentation_url: URL to error documentation
+    """
+    code: str
+    title: str
+    description: str
+    documentation_url: Optional[str]
+

supadata-1.0.0/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for the Supadata package.""" 

supadata-1.0.0/tests/test_client.py

@@ -0,0 +1,180 @@
+"""Tests for the Supadata client."""
+
+import pytest
+import requests
+from requests import Response
+
+from supadata import (
+    Supadata,
+    Transcript,
+    TranslatedTranscript,
+    TranscriptChunk,
+    Scrape,
+    Map,
+    Error,
+)
+
+
+@pytest.fixture
+def api_key() -> str:
+    """Return a dummy API key for testing."""
+    return "test_api_key"
+
+
+@pytest.fixture
+def base_url() -> str:
+    """Return a dummy base URL for testing."""
+    return "https://api.test.com/v1"
+
+
+@pytest.fixture
+def client(api_key: str, base_url: str) -> Supadata:
+    """Return a configured Supadata client."""
+    return Supadata(api_key=api_key, base_url=base_url)
+
+
+def test_client_initialization(api_key: str, base_url: str) -> None:
+    """Test client initialization."""
+    client = Supadata(api_key=api_key, base_url=base_url)
+    assert client.base_url == base_url
+    assert client.session.headers["x-api-key"] == api_key
+    assert client.session.headers["Accept"] == "application/json"
+
+
+def test_get_transcript_chunks(client: Supadata, requests_mock) -> None:
+    """Test getting YouTube transcript with chunks."""
+    video_id = "test123"
+    mock_response = {
+        "content": [
+            {
+                "text": "Hello",
+                "offset": 0,
+                "duration": 1000,
+                "lang": "en"
+            }
+        ],
+        "lang": "en",
+        "availableLangs": ["en", "es"]
+    }
+    requests_mock.get(
+        f"{client.base_url}/youtube/transcript",
+        json=mock_response
+    )
+
+    transcript = client.get_transcript(video_id=video_id)
+    assert isinstance(transcript, Transcript)
+    assert isinstance(transcript.content[0], TranscriptChunk)
+    assert transcript.content[0].text == "Hello"
+    assert transcript.lang == "en"
+    assert transcript.available_langs == ["en", "es"]
+
+
+def test_get_transcript_text(client: Supadata, requests_mock) -> None:
+    """Test getting YouTube transcript as plain text."""
+    video_id = "test123"
+    mock_response = {
+        "content": "Hello, this is a test transcript",
+        "lang": "en",
+        "availableLangs": ["en", "es"]
+    }
+    requests_mock.get(
+        f"{client.base_url}/youtube/transcript",
+        json=mock_response
+    )
+
+    transcript = client.get_transcript(video_id=video_id, text=True)
+    assert isinstance(transcript, Transcript)
+    assert isinstance(transcript.content, str)
+    assert transcript.content == "Hello, this is a test transcript"
+
+
+def test_translate_transcript(client: Supadata, requests_mock) -> None:
+    """Test translating YouTube transcript."""
+    video_id = "test123"
+    mock_response = {
+        "content": "Hola, esto es una prueba",
+        "lang": "es"
+    }
+    requests_mock.get(
+        f"{client.base_url}/youtube/transcript/translate",
+        json=mock_response
+    )
+
+    transcript = client.translate_transcript(
+        video_id=video_id,
+        lang="es",
+        text=True
+    )
+    assert isinstance(transcript, TranslatedTranscript)
+    assert transcript.content == "Hola, esto es una prueba"
+    assert transcript.lang == "es"
+
+
+def test_scrape(client: Supadata, requests_mock) -> None:
+    """Test web scraping."""
+    url = "https://test.com"
+    mock_response = {
+        "url": url,
+        "content": "# Test\nThis is a test page",
+        "name": "Test Page",
+        "description": "A test page",
+        "ogUrl": "https://test.com/og.png",
+        "countCharacters": 100,
+        "urls": ["https://test.com/about"]
+    }
+    requests_mock.get(
+        f"{client.base_url}/web/scrape",
+        json=mock_response
+    )
+
+    content = client.scrape(url=url)
+    assert isinstance(content, Scrape)
+    assert content.url == url
+    assert content.name == "Test Page"
+    assert content.og_url == "https://test.com/og.png"
+    assert content.count_characters == 100
+
+
+def test_map(client: Supadata, requests_mock) -> None:
+    """Test site mapping."""
+    url = "https://test.com"
+    mock_response = {
+        "urls": [
+            "https://test.com",
+            "https://test.com/about"
+        ]
+    }
+    requests_mock.get(
+        f"{client.base_url}/web/map",
+        json=mock_response
+    )
+
+    site_map = client.map(url=url)
+    assert isinstance(site_map, Map)
+    assert len(site_map.urls) == 2
+
+
+def test_error_handling(client: Supadata, requests_mock) -> None:
+    """Test error handling."""
+    video_id = "invalid"
+    error_response = {
+        "code": "video-not-found",
+        "title": "Video Not Found",
+        "description": "The specified video was not found",
+        "documentationUrl": "https://docs.test.com/errors#video-not-found"
+    }
+    requests_mock.get(
+        f"{client.base_url}/youtube/transcript",
+        status_code=404,
+        json=error_response
+    )
+
+    with pytest.raises(requests.exceptions.HTTPError) as exc_info:
+        client.get_transcript(video_id=video_id)
+
+    error = exc_info.value.args[0]
+    assert isinstance(error, Error)
+    assert error.code == error_response["code"]
+    assert error.title == error_response["title"]
+    assert error.description == error_response["description"]
+    assert error.documentation_url == error_response["documentationUrl"]