nstdata-ai-crawl 0.1.0__tar.gz

nstdata_ai_crawl-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 nstdata
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nstdata_ai_crawl-0.1.0/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: nstdata-ai-crawl
+Version: 0.1.0
+Summary: Python SDK for nstdata
+Home-page: https://github.com/nstdata-ai/crawl-py
+Author: nstdata-ai
+Author-email: github@nstproxy.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.0.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# nstdata crawl SDK (Python)
+
+[中文版](README_zh.md)
+
+Python SDK for nstdata crawl, an advanced web scraping and crawling service.
+
+## Installation
+
+```bash
+pip install nstdata-ai-crawl
+```
+
+## Features
+
+- **Basic Scraping:** Get HTML and Markdown synchronously.
+- **Screenshots & PDFs:** Take full-page screenshots and export pages as PDFs.
+- **Browser Actions:** Orchestrate browser actions like wait for elements, fill forms, click buttons, scroll, and execute JS.
+- **Custom Configuration:** Bring your own cookies, headers, and simulate mobile devices.
+- **Content Filtering:** Automatically extract high-quality main content, filter out ads and navigational elements.
+- **Proxy Support:** Specify proxy type (residential or datacenter) and exit country.
+- **Asynchronous Execution & Polling:** Submit time-consuming tasks and poll for results.
+- **Batch Crawling:** Traverse entire sites with depth and page limits.
+
+## Quick Start
+
+See [main.py](./main.py) for a complete list of usage examples.
+
+```python
+import os
+from nstdata_ai_crawl import NstDataClient, ScrapeRequestDto, Format
+
+# Initialize the client with your API token
+TOKEN = os.getenv("NSTDATA_API_TOKEN", "YOUR_API_TOKEN")
+
+def main():
+    with NstDataClient(TOKEN) as client:
+        # Submit a basic scraping task
+        res = client.submit_scrape_task_sync(ScrapeRequestDto(
+            url="https://example.com/",
+            formats=[Format.MARKDOWN],
+            timeout=60000,
+        ))
+        
+        # Output the resulting markdown
+        print(res.data.get_markdown())
+
+if __name__ == "__main__":
+    main()
+```

nstdata_ai_crawl-0.1.0/README.md

@@ -0,0 +1,49 @@
+# nstdata crawl SDK (Python)
+
+[中文版](README_zh.md)
+
+Python SDK for nstdata crawl, an advanced web scraping and crawling service.
+
+## Installation
+
+```bash
+pip install nstdata-ai-crawl
+```
+
+## Features
+
+- **Basic Scraping:** Get HTML and Markdown synchronously.
+- **Screenshots & PDFs:** Take full-page screenshots and export pages as PDFs.
+- **Browser Actions:** Orchestrate browser actions like wait for elements, fill forms, click buttons, scroll, and execute JS.
+- **Custom Configuration:** Bring your own cookies, headers, and simulate mobile devices.
+- **Content Filtering:** Automatically extract high-quality main content, filter out ads and navigational elements.
+- **Proxy Support:** Specify proxy type (residential or datacenter) and exit country.
+- **Asynchronous Execution & Polling:** Submit time-consuming tasks and poll for results.
+- **Batch Crawling:** Traverse entire sites with depth and page limits.
+
+## Quick Start
+
+See [main.py](./main.py) for a complete list of usage examples.
+
+```python
+import os
+from nstdata_ai_crawl import NstDataClient, ScrapeRequestDto, Format
+
+# Initialize the client with your API token
+TOKEN = os.getenv("NSTDATA_API_TOKEN", "YOUR_API_TOKEN")
+
+def main():
+    with NstDataClient(TOKEN) as client:
+        # Submit a basic scraping task
+        res = client.submit_scrape_task_sync(ScrapeRequestDto(
+            url="https://example.com/",
+            formats=[Format.MARKDOWN],
+            timeout=60000,
+        ))
+        
+        # Output the resulting markdown
+        print(res.data.get_markdown())
+
+if __name__ == "__main__":
+    main()
+```

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl/__init__.py

@@ -0,0 +1,37 @@
+from .client import NstDataClient
+from .constants import Format, TaskType, TaskStatus, ActionType, Proxy
+from .models import (
+    CookieDto,
+    ActionDto,
+    JsRenderDto,
+    ScrapeRequestDto,
+    CrawlRequestDto,
+    SubmitResponseDto,
+    PageMetadataDto,
+    ScrapeDataDto,
+    TaskResultResponseDto,
+    CrawlStatusResponseDto,
+    CrawlPageItemResponseDto,
+    CrawlPageListResponseDto
+)
+
+__all__ = [
+    "NstDataClient",
+    "Format",
+    "TaskType",
+    "TaskStatus",
+    "ActionType",
+    "Proxy",
+    "CookieDto",
+    "ActionDto",
+    "JsRenderDto",
+    "ScrapeRequestDto",
+    "CrawlRequestDto",
+    "SubmitResponseDto",
+    "PageMetadataDto",
+    "ScrapeDataDto",
+    "TaskResultResponseDto",
+    "CrawlStatusResponseDto",
+    "CrawlPageItemResponseDto",
+    "CrawlPageListResponseDto"
+]

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl/client.py

@@ -0,0 +1,149 @@
+import requests
+import base64
+from typing import Optional, Dict
+from .models import (
+    ScrapeRequestDto,
+    CrawlRequestDto,
+    SubmitResponseDto,
+    TaskResultResponseDto,
+    PageMetadataDto,
+    ScrapeDataDto,
+    CrawlStatusResponseDto,
+    CrawlPageItemResponseDto,
+    CrawlPageListResponseDto
+)
+
+# Built-in API endpoint.
+_BASE_URL = "https://api.nstproxy.com"
+
+def _build(cls, data: dict):
+    """Construct a DTO from a response dict, ignoring keys the DTO doesn't
+    declare so server-side field additions don't break the client."""
+    return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
+
+class NstDataClient:
+    def __init__(self, token: str):
+        self.base_url = _BASE_URL
+        self.token = token
+        self.session = requests.Session()
+
+    def _get_headers(self) -> Dict[str, str]:
+        return {"x-api-key": self.token}
+
+    def _unwrap(self, resp: requests.Response) -> dict:
+        resp.raise_for_status()
+        data = resp.json()
+        if isinstance(data, dict) and "err" in data and "code" in data:
+            if data.get("err") or data.get("code") != 200:
+                raise Exception(f"API error: code={data.get('code')} msg={data.get('msg')}")
+            return data.get("data") or {}
+        return data
+
+    def submit_scrape_task(self, req: ScrapeRequestDto) -> SubmitResponseDto:
+        if not req.url:
+            raise ValueError("url is required")
+        if not req.formats:
+            raise ValueError("formats must contain at least one format")
+        url = f"{self.base_url}/api/v1/crawl/scrape/submit"
+        resp = self.session.post(url, json=req.to_dict(), headers=self._get_headers())
+        data = self._unwrap(resp)
+        return _build(SubmitResponseDto, data)
+
+    def submit_scrape_task_sync(self, req: ScrapeRequestDto) -> TaskResultResponseDto:
+        if not req.url:
+            raise ValueError("url is required")
+        if not req.formats:
+            raise ValueError("formats must contain at least one format")
+        url = f"{self.base_url}/api/v1/crawl/scrape/submit-sync"
+        resp = self.session.post(url, json=req.to_dict(), headers=self._get_headers())
+        data = self._unwrap(resp)
+        
+        if 'data' in data and data['data'] is not None:
+            if 'metadata' in data['data'] and data['data']['metadata'] is not None:
+                data['data']['metadata'] = _build(PageMetadataDto, data['data']['metadata'])
+            raw = data['data']
+            data['data'] = _build(ScrapeDataDto, raw)
+            data['data']._html = raw.get('html')
+            data['data']._markdown = raw.get('markdown')
+            data['data']._client = self
+            
+        return _build(TaskResultResponseDto, data)
+
+    def submit_crawl_task(self, req: CrawlRequestDto) -> SubmitResponseDto:
+        if not req.url:
+            raise ValueError("url is required")
+        if not req.formats:
+            raise ValueError("formats must contain at least one format")
+        url = f"{self.base_url}/api/v1/crawl/submit"
+        resp = self.session.post(url, json=req.to_dict(), headers=self._get_headers())
+        data = self._unwrap(resp)
+        return _build(SubmitResponseDto, data)
+
+    def get_crawl_status(self, crawl_id: str) -> CrawlStatusResponseDto:
+        url = f"{self.base_url}/api/v1/crawl/status/{crawl_id}"
+        resp = self.session.get(url, headers=self._get_headers())
+        data = self._unwrap(resp)
+        return _build(CrawlStatusResponseDto, data)
+
+    def list_crawl_pages(self, crawl_id: str, after_cursor: Optional[str] = None, before_cursor: Optional[str] = None, limit: Optional[int] = None) -> CrawlPageListResponseDto:
+        url = f"{self.base_url}/api/v1/crawl/pages/{crawl_id}"
+        params = {}
+        if after_cursor is not None:
+            params["after_cursor"] = after_cursor
+        if before_cursor is not None:
+            params["before_cursor"] = before_cursor
+        if limit is not None:
+            params["limit"] = limit
+
+        resp = self.session.get(url, params=params, headers=self._get_headers())
+        data = self._unwrap(resp)
+
+        if 'data' in data and data['data'] is not None:
+            items = []
+            for item in data['data']:
+                if item.get('data') is not None:
+                    if item['data'].get('metadata') is not None:
+                        item['data']['metadata'] = _build(PageMetadataDto, item['data']['metadata'])
+                    raw = item['data']
+                    item['data'] = _build(ScrapeDataDto, raw)
+                    item['data']._html = raw.get('html')
+                    item['data']._markdown = raw.get('markdown')
+                    
+                    item['data']._client = self
+                items.append(_build(CrawlPageItemResponseDto, item))
+            data['data'] = items
+
+        return _build(CrawlPageListResponseDto, data)
+
+    def get_scrape_task_result(self, task_id: str) -> TaskResultResponseDto:
+        url = f"{self.base_url}/api/v1/crawl/scrape/result/{task_id}"
+        resp = self.session.get(url, headers=self._get_headers())
+        data = self._unwrap(resp)
+        
+        if 'data' in data and data['data'] is not None:
+            if 'metadata' in data['data'] and data['data']['metadata'] is not None:
+                data['data']['metadata'] = _build(PageMetadataDto, data['data']['metadata'])
+            raw = data['data']
+            data['data'] = _build(ScrapeDataDto, raw)
+            data['data']._html = raw.get('html')
+            data['data']._markdown = raw.get('markdown')
+            data['data']._client = self
+            
+        return _build(TaskResultResponseDto, data)
+
+    def read_storage(self, st: str) -> bytes:
+        url = f"{self.base_url}/api/v1/crawl/storage/read"
+        resp = self.session.get(url, params={"st": st}, headers=self._get_headers())
+        resp.raise_for_status()
+        return resp.content
+
+
+
+    def close(self):
+        self.session.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl/constants.py

@@ -0,0 +1,34 @@
+class Format:
+    HTML = "html"
+    MARKDOWN = "markdown"
+    LINKS = "links"
+    PDF = "pdf"
+    SCREENSHOT = "screenshot"
+    RAW_DATA = "rawData"
+
+class TaskType:
+    SCRAPE = "scrape"
+    CRAWL = "crawl"
+
+class TaskStatus:
+    PROCESSING = "processing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    SCRAPING = "scraping"  # crawl 会话仍在抓取中
+
+class ActionType:
+    CLICK = "click"                                          # click an element; requires selector
+    WAIT = "wait"                                            # wait a fixed time; requires milliseconds
+    WAIT_SELECTOR = "waitSelector"                           # wait until element is visible; requires selector
+    SCROLL_X = "scrollX"                                     # scroll horizontally by x pixels
+    SCROLL_Y = "scrollY"                                     # scroll vertically by y pixels
+    SCROLL_TO = "scrollTo"                                   # scroll to an element; requires selector
+    FILL = "fill"                                            # fill an input; requires selector and text
+    EXECUTE = "execute"                                      # run JavaScript on the page; requires execute
+    SCREENSHOT = "screenshot"                                # capture screenshot; fullScreenshot / particularScreenshot / viewport
+    WAIT_FOR_REQUEST_COMPLETION = "waitForRequestCompletion" # wait for a matching network response; requires urlPattern
+
+class Proxy:
+    """Proxy types for ScrapeRequestDto.proxy (defaults to residential when unset)."""
+    RESIDENTIAL = "residential"
+    DATACENTER = "datacenter"

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl/models.py

@@ -0,0 +1,264 @@
+from typing import Optional, List, Dict, Any
+from dataclasses import dataclass, asdict, field
+
+from .constants import ActionType
+
+
+def _to_clean_dict(obj) -> Dict[str, Any]:
+    def _clean(v):
+        if isinstance(v, dict):
+            return {k: _clean(x) for k, x in v.items() if x is not None}
+        elif isinstance(v, list):
+            return [_clean(x) for x in v if x is not None]
+        return v
+    return _clean(asdict(obj))
+
+@dataclass
+class CookieDto:
+    domain: Optional[str] = None # Domain the cookie belongs to
+    httpOnly: Optional[bool] = None # True if the cookie is HTTP-only
+    name: Optional[str] = None # Name of the cookie
+    path: Optional[str] = None # Path for the cookie
+    secure: Optional[bool] = None # True if the cookie requires HTTPS
+    value: Optional[str] = None # Value of the cookie
+
+@dataclass
+class ActionDto:
+    execute: Optional[str] = None # JavaScript code snippet to execute
+    fullScreenshot: Optional[bool] = None # Set to true to capture a full-page screenshot
+    key: Optional[str] = None # Keyboard key to press
+    milliseconds: Optional[float] = None # Milliseconds to wait
+    particularScreenshot: Optional[str] = None # CSS selector of an element to screenshot
+    selector: Optional[str] = None # CSS selector for the action target
+    text: Optional[str] = None # Text to type into an input field
+    type: Optional[str] = None # Type of action (e.g., 'click', 'wait', 'fill')
+    urlPattern: Optional[str] = None # URL pattern to wait for (regex support)
+    waitSelector: Optional[str] = None # CSS selector to wait for
+    x: Optional[float] = None # X coordinate for scroll/click
+    y: Optional[float] = None # Y coordinate for scroll/click
+
+    # Factory helpers: each one sets `type` plus the fields that action
+    # requires, so callers don't need to remember which fields pair with
+    # which type.
+
+    @classmethod
+    def click(cls, selector: str) -> "ActionDto":
+        """Click the element matched by selector."""
+        return cls(type=ActionType.CLICK, selector=selector)
+
+    @classmethod
+    def wait(cls, milliseconds: float) -> "ActionDto":
+        """Pause for a fixed time in milliseconds."""
+        return cls(type=ActionType.WAIT, milliseconds=milliseconds)
+
+    @classmethod
+    def wait_selector(cls, selector: str, timeout_ms: Optional[float] = None) -> "ActionDto":
+        """Wait until the element matched by selector is visible."""
+        return cls(type=ActionType.WAIT_SELECTOR, selector=selector, milliseconds=timeout_ms)
+
+    @classmethod
+    def scroll_x(cls, pixels: int) -> "ActionDto":
+        """Scroll the window horizontally by pixels."""
+        return cls(type=ActionType.SCROLL_X, text=str(pixels), x=pixels)
+
+    @classmethod
+    def scroll_y(cls, pixels: int) -> "ActionDto":
+        """Scroll the window vertically by pixels."""
+        return cls(type=ActionType.SCROLL_Y, text=str(pixels), y=pixels)
+
+    @classmethod
+    def scroll_to(cls, selector: str) -> "ActionDto":
+        """Scroll to the element matched by selector."""
+        return cls(type=ActionType.SCROLL_TO, selector=selector)
+
+    @classmethod
+    def fill(cls, selector: str, text: str) -> "ActionDto":
+        """Type text into the input matched by selector."""
+        return cls(type=ActionType.FILL, selector=selector, text=text)
+
+    @classmethod
+    def execute_js(cls, script: str) -> "ActionDto":
+        """Run a JavaScript snippet on the page."""
+        return cls(type=ActionType.EXECUTE, execute=script)
+
+    @classmethod
+    def screenshot(cls, full: bool = False, selector: Optional[str] = None) -> "ActionDto":
+        """Capture a screenshot: full page if full=True, the element matched
+        by selector if given, otherwise the current viewport."""
+        return cls(type=ActionType.SCREENSHOT, fullScreenshot=full or None, particularScreenshot=selector)
+
+    @classmethod
+    def wait_for_request(cls, url_pattern: str, timeout_ms: Optional[float] = None) -> "ActionDto":
+        """Wait for a network response whose URL matches url_pattern
+        (supports * wildcard)."""
+        return cls(type=ActionType.WAIT_FOR_REQUEST_COMPLETION, urlPattern=url_pattern, milliseconds=timeout_ms)
+
+@dataclass
+class JsRenderDto:
+    actions: Optional[List[ActionDto]] = None # List of browser actions to perform
+    waitFor: Optional[float] = None # Global wait time in milliseconds
+    waitForSelector: Optional[str] = None # CSS selector to wait for before proceeding
+
+@dataclass
+class ScrapeRequestDto:
+    url: str # The URL to scrape
+    cookies: Optional[List[CookieDto]] = None # Optional cookies to include in the request
+    country: Optional[str] = None # Country code for proxy location (e.g., 'us', 'gb')
+    excludeTags: Optional[List[str]] = None # HTML tags to remove entirely (e.g., ['nav', 'footer'])
+    formats: Optional[List[str]] = None # Desired output formats (e.g., ['markdown', 'html', 'links'])
+    headers: Optional[Dict[str, str]] = None # Custom HTTP headers
+    includeTags: Optional[List[str]] = None # Tags to keep. WARNING: strips all other tags (keeping text), which may break markdown
+    jsRender: Optional[JsRenderDto] = None # JavaScript rendering instructions
+    mobile: Optional[bool] = None # Whether to emulate a mobile device
+    onlyMainContent: Optional[bool] = None # Automatically extract main article content and remove noise
+    proxy: Optional[str] = None # Proxy type (e.g., 'datacenter', 'residential')
+    removeSelectors: Optional[List[str]] = None # CSS selectors for elements to remove (e.g., ['.ads', '#popup'])
+    targetSelector: Optional[str] = None # CSS selector to restrict extraction to a specific element
+    timeout: Optional[float] = None # Request timeout in milliseconds
+
+    def to_dict(self) -> Dict[str, Any]:
+        return _to_clean_dict(self)
+
+@dataclass
+class CrawlRequestDto:
+    """Scrape options plus crawl traversal controls
+    (maxDepth, maxPages, includeUrls, excludeUrls, ignoreQuery)."""
+    url: str # The starting URL to crawl
+    cookies: Optional[List[CookieDto]] = None # Optional cookies to include in the request
+    country: Optional[str] = None # Country code for proxy location
+    excludeTags: Optional[List[str]] = None # HTML tags to remove entirely
+    excludeUrls: Optional[List[str]] = None # Regex patterns to exclude specific URLs from crawling
+    formats: Optional[List[str]] = None # Desired output formats (e.g., ['markdown', 'html'])
+    headers: Optional[Dict[str, str]] = None # Custom HTTP headers
+    ignoreQuery: Optional[bool] = None # Whether to ignore URL query parameters for deduplication
+    includeTags: Optional[List[str]] = None # Tags to keep. WARNING: strips all other tags
+    includeUrls: Optional[List[str]] = None # Regex patterns to only crawl matching URLs
+    jsRender: Optional[JsRenderDto] = None # JavaScript rendering instructions
+    maxDepth: Optional[int] = None # Maximum depth of links to follow
+    maxPages: Optional[int] = None # Maximum number of pages to crawl
+    mobile: Optional[bool] = None # Whether to emulate a mobile device
+    onlyMainContent: Optional[bool] = None # Automatically extract main article content
+    proxy: Optional[str] = None # Proxy type
+    removeSelectors: Optional[List[str]] = None # CSS selectors for elements to remove
+    targetSelector: Optional[str] = None # CSS selector to restrict extraction
+    timeout: Optional[float] = None # Request timeout in milliseconds
+
+    def to_dict(self) -> Dict[str, Any]:
+        return _to_clean_dict(self)
+
+@dataclass
+class SubmitResponseDto:
+    cached: Optional[bool] = None # True if the response was served from cache
+    code: Optional[int] = None # HTTP status code
+    id: Optional[str] = None # Task ID
+    status: Optional[str] = None # Task status message
+
+@dataclass
+class PageMetadataDto:
+    canonical: Optional[str] = None # Canonical URL of the page
+    description: Optional[str] = None # Meta description text
+    language: Optional[str] = None # Document language code
+    ogImage: Optional[str] = None # OpenGraph preview image URL
+    statusCode: Optional[int] = None # HTTP response status code
+    title: Optional[str] = None # Page title
+
+@dataclass
+class ScrapeDataDto:
+    htmlRef: Optional[str] = None # Reference link to fetch the HTML content
+    links: Optional[List[str]] = None # List of extracted hyperlinks
+    markdownRef: Optional[str] = None # Reference link to fetch the Markdown content
+    metadata: Optional[PageMetadataDto] = None # Extracted page metadata and SEO tags
+    pdfRef: Optional[str] = None # Reference link to fetch the PDF
+    rawDataRef: Optional[str] = None # Reference link to fetch the raw data
+    screenshotRef: Optional[str] = None # Reference link to fetch the screenshot
+
+
+    _html: Optional[str] = field(default=None, repr=False, init=False)
+    _markdown: Optional[str] = field(default=None, repr=False, init=False)
+    _pdf: Optional[bytes] = field(default=None, repr=False, init=False)
+    _rawData: Optional[bytes] = field(default=None, repr=False, init=False)
+    _screenshot: Optional[bytes] = field(default=None, repr=False, init=False)
+    _client: Any = field(default=None, repr=False, init=False)
+
+    def get_html(self) -> Optional[str]:
+        if self._html is not None:
+            return self._html
+        if self.htmlRef and self._client:
+            self._html = self._client.read_storage(self.htmlRef).decode('utf-8')
+            return self._html
+        return None
+
+    def get_markdown(self) -> Optional[str]:
+        if self._markdown is not None:
+            return self._markdown
+        if self.markdownRef and self._client:
+            self._markdown = self._client.read_storage(self.markdownRef).decode('utf-8')
+            return self._markdown
+        return None
+
+    def get_pdf(self) -> Optional[bytes]:
+        if self._pdf is not None:
+            return self._pdf
+        if self.pdfRef and self._client:
+            self._pdf = self._client.read_storage(self.pdfRef)
+            return self._pdf
+        return None
+
+    def get_raw_data(self) -> Optional[bytes]:
+        if self._rawData is not None:
+            return self._rawData
+        if self.rawDataRef and self._client:
+            self._rawData = self._client.read_storage(self.rawDataRef)
+            return self._rawData
+        return None
+
+    def get_screenshot(self) -> Optional[bytes]:
+        if self._screenshot is not None:
+            return self._screenshot
+        if self.screenshotRef and self._client:
+            self._screenshot = self._client.read_storage(self.screenshotRef)
+            return self._screenshot
+        return None
+
+@dataclass
+class TaskResultResponseDto:
+    code: Optional[int] = None # Custom status code
+    data: Optional[ScrapeDataDto] = None # The extracted scrape/crawl data
+    errorCode: Optional[str] = None # Error code identifier if any
+    errorMessage: Optional[str] = None # Detailed error message
+    status: Optional[str] = None # Current status (e.g., 'completed', 'failed')
+    success: Optional[bool] = None # True if the task executed successfully
+
+@dataclass
+class CrawlStatusResponseDto:
+    """Aggregate progress of a crawl session."""
+    code: Optional[int] = None # Custom status code
+    completed: Optional[int] = None # Number of completed pages
+    failed: Optional[int] = None # Number of failed pages
+    id: Optional[str] = None # Crawl task ID
+    pending: Optional[int] = None # Number of pending pages
+    status: Optional[str] = None # Current status message
+    total: Optional[int] = None # Total number of discovered pages
+
+@dataclass
+class CrawlPageItemResponseDto:
+    """One page's normalized result within a crawl session."""
+    data: Optional[ScrapeDataDto] = None # Extracted data for this page
+    errorCode: Optional[str] = None # Error code if any
+    errorMessage: Optional[str] = None # Detailed error message
+    status: Optional[str] = None # Status of this page extraction
+    success: Optional[bool] = None # True if extraction was successful
+    taskId: Optional[str] = None # ID of the specific page task
+    url: Optional[str] = None # URL of the scraped page
+
+@dataclass
+class CrawlPageListResponseDto:
+    """Cursor-paginated list of crawl page results."""
+    code: Optional[int] = None # Custom status code
+    completed: Optional[int] = None # Number of completed pages
+    data: Optional[List[CrawlPageItemResponseDto]] = None # List of page results
+    nextCursor: Optional[str] = None # Cursor for fetching the next page of results
+    prevCursor: Optional[str] = None # Cursor for fetching the previous page of results
+    total: Optional[int] = None # Total number of pages
+
+

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl.egg-info/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: nstdata-ai-crawl
+Version: 0.1.0
+Summary: Python SDK for nstdata
+Home-page: https://github.com/nstdata-ai/crawl-py
+Author: nstdata-ai
+Author-email: github@nstproxy.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.0.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# nstdata crawl SDK (Python)
+
+[中文版](README_zh.md)
+
+Python SDK for nstdata crawl, an advanced web scraping and crawling service.
+
+## Installation
+
+```bash
+pip install nstdata-ai-crawl
+```
+
+## Features
+
+- **Basic Scraping:** Get HTML and Markdown synchronously.
+- **Screenshots & PDFs:** Take full-page screenshots and export pages as PDFs.
+- **Browser Actions:** Orchestrate browser actions like wait for elements, fill forms, click buttons, scroll, and execute JS.
+- **Custom Configuration:** Bring your own cookies, headers, and simulate mobile devices.
+- **Content Filtering:** Automatically extract high-quality main content, filter out ads and navigational elements.
+- **Proxy Support:** Specify proxy type (residential or datacenter) and exit country.
+- **Asynchronous Execution & Polling:** Submit time-consuming tasks and poll for results.
+- **Batch Crawling:** Traverse entire sites with depth and page limits.
+
+## Quick Start
+
+See [main.py](./main.py) for a complete list of usage examples.
+
+```python
+import os
+from nstdata_ai_crawl import NstDataClient, ScrapeRequestDto, Format
+
+# Initialize the client with your API token
+TOKEN = os.getenv("NSTDATA_API_TOKEN", "YOUR_API_TOKEN")
+
+def main():
+    with NstDataClient(TOKEN) as client:
+        # Submit a basic scraping task
+        res = client.submit_scrape_task_sync(ScrapeRequestDto(
+            url="https://example.com/",
+            formats=[Format.MARKDOWN],
+            timeout=60000,
+        ))
+        
+        # Output the resulting markdown
+        print(res.data.get_markdown())
+
+if __name__ == "__main__":
+    main()
+```

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+README.md
+pyproject.toml
+setup.py
+nstdata_ai_crawl/__init__.py
+nstdata_ai_crawl/client.py
+nstdata_ai_crawl/constants.py
+nstdata_ai_crawl/models.py
+nstdata_ai_crawl.egg-info/PKG-INFO
+nstdata_ai_crawl.egg-info/SOURCES.txt
+nstdata_ai_crawl.egg-info/dependency_links.txt
+nstdata_ai_crawl.egg-info/requires.txt
+nstdata_ai_crawl.egg-info/top_level.txt

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl.egg-info/requires.txt

@@ -0,0 +1 @@
+requests>=2.0.0

nstdata_ai_crawl-0.1.0/nstdata_ai_crawl.egg-info/top_level.txt

@@ -0,0 +1 @@
+nstdata_ai_crawl

nstdata_ai_crawl-0.1.0/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"

nstdata_ai_crawl-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

nstdata_ai_crawl-0.1.0/setup.py

@@ -0,0 +1,26 @@
+from setuptools import setup, find_packages
+from pathlib import Path
+
+this_directory = Path(__file__).parent
+long_description = (this_directory / "README.md").read_text(encoding="utf-8")
+
+setup(
+    name="nstdata-ai-crawl",
+    version="0.1.0",
+    packages=find_packages(),
+    install_requires=[
+        "requests>=2.0.0",
+    ],
+    author="nstdata-ai",
+    author_email="github@nstproxy.com",
+    description="Python SDK for nstdata",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/nstdata-ai/crawl-py",
+    python_requires=">=3.7",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+)