cartography-client 0.0.1__py3-none-any.whl

cartography/resources/workflows/request/request.py

@@ -0,0 +1,221 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import List
+
+import httpx
+
+from .crawl import (
+    CrawlResource,
+    AsyncCrawlResource,
+    CrawlResourceWithRawResponse,
+    AsyncCrawlResourceWithRawResponse,
+    CrawlResourceWithStreamingResponse,
+    AsyncCrawlResourceWithStreamingResponse,
+)
+from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven
+from ...._utils import maybe_transform, async_maybe_transform
+from ...._compat import cached_property
+from ...._resource import SyncAPIResource, AsyncAPIResource
+from ...._response import (
+    to_raw_response_wrapper,
+    to_streamed_response_wrapper,
+    async_to_raw_response_wrapper,
+    async_to_streamed_response_wrapper,
+)
+from ...._base_client import make_request_options
+from ....types.workflows import request_create_download_params
+from ....types.workflows.request_create_download_response import RequestCreateDownloadResponse
+
+__all__ = ["RequestResource", "AsyncRequestResource"]
+
+
+class RequestResource(SyncAPIResource):
+    @cached_property
+    def crawl(self) -> CrawlResource:
+        return CrawlResource(self._client)
+
+    @cached_property
+    def with_raw_response(self) -> RequestResourceWithRawResponse:
+        """
+        This property can be used as a prefix for any HTTP method call to return
+        the raw response object instead of the parsed content.
+
+        For more information, see https://www.github.com/evrimai/cartography-client#accessing-raw-response-data-eg-headers
+        """
+        return RequestResourceWithRawResponse(self)
+
+    @cached_property
+    def with_streaming_response(self) -> RequestResourceWithStreamingResponse:
+        """
+        An alternative to `.with_raw_response` that doesn't eagerly read the response body.
+
+        For more information, see https://www.github.com/evrimai/cartography-client#with_streaming_response
+        """
+        return RequestResourceWithStreamingResponse(self)
+
+    def create_download(
+        self,
+        *,
+        bucket_name: str,
+        crawl_id: str,
+        downloader_type: str,
+        urls: List[str],
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> RequestCreateDownloadResponse:
+        """
+        Make a request to temporal worker :param request: crawl request data :param
+        token_data: api token :return: response
+
+        Args:
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        return self._post(
+            "/workflows/request/download",
+            body=maybe_transform(
+                {
+                    "bucket_name": bucket_name,
+                    "crawl_id": crawl_id,
+                    "downloader_type": downloader_type,
+                    "urls": urls,
+                },
+                request_create_download_params.RequestCreateDownloadParams,
+            ),
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=RequestCreateDownloadResponse,
+        )
+
+
+class AsyncRequestResource(AsyncAPIResource):
+    @cached_property
+    def crawl(self) -> AsyncCrawlResource:
+        return AsyncCrawlResource(self._client)
+
+    @cached_property
+    def with_raw_response(self) -> AsyncRequestResourceWithRawResponse:
+        """
+        This property can be used as a prefix for any HTTP method call to return
+        the raw response object instead of the parsed content.
+
+        For more information, see https://www.github.com/evrimai/cartography-client#accessing-raw-response-data-eg-headers
+        """
+        return AsyncRequestResourceWithRawResponse(self)
+
+    @cached_property
+    def with_streaming_response(self) -> AsyncRequestResourceWithStreamingResponse:
+        """
+        An alternative to `.with_raw_response` that doesn't eagerly read the response body.
+
+        For more information, see https://www.github.com/evrimai/cartography-client#with_streaming_response
+        """
+        return AsyncRequestResourceWithStreamingResponse(self)
+
+    async def create_download(
+        self,
+        *,
+        bucket_name: str,
+        crawl_id: str,
+        downloader_type: str,
+        urls: List[str],
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> RequestCreateDownloadResponse:
+        """
+        Make a request to temporal worker :param request: crawl request data :param
+        token_data: api token :return: response
+
+        Args:
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        return await self._post(
+            "/workflows/request/download",
+            body=await async_maybe_transform(
+                {
+                    "bucket_name": bucket_name,
+                    "crawl_id": crawl_id,
+                    "downloader_type": downloader_type,
+                    "urls": urls,
+                },
+                request_create_download_params.RequestCreateDownloadParams,
+            ),
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=RequestCreateDownloadResponse,
+        )
+
+
+class RequestResourceWithRawResponse:
+    def __init__(self, request: RequestResource) -> None:
+        self._request = request
+
+        self.create_download = to_raw_response_wrapper(
+            request.create_download,
+        )
+
+    @cached_property
+    def crawl(self) -> CrawlResourceWithRawResponse:
+        return CrawlResourceWithRawResponse(self._request.crawl)
+
+
+class AsyncRequestResourceWithRawResponse:
+    def __init__(self, request: AsyncRequestResource) -> None:
+        self._request = request
+
+        self.create_download = async_to_raw_response_wrapper(
+            request.create_download,
+        )
+
+    @cached_property
+    def crawl(self) -> AsyncCrawlResourceWithRawResponse:
+        return AsyncCrawlResourceWithRawResponse(self._request.crawl)
+
+
+class RequestResourceWithStreamingResponse:
+    def __init__(self, request: RequestResource) -> None:
+        self._request = request
+
+        self.create_download = to_streamed_response_wrapper(
+            request.create_download,
+        )
+
+    @cached_property
+    def crawl(self) -> CrawlResourceWithStreamingResponse:
+        return CrawlResourceWithStreamingResponse(self._request.crawl)
+
+
+class AsyncRequestResourceWithStreamingResponse:
+    def __init__(self, request: AsyncRequestResource) -> None:
+        self._request = request
+
+        self.create_download = async_to_streamed_response_wrapper(
+            request.create_download,
+        )
+
+    @cached_property
+    def crawl(self) -> AsyncCrawlResourceWithStreamingResponse:
+        return AsyncCrawlResourceWithStreamingResponse(self._request.crawl)

cartography/resources/workflows/workflows.py

@@ -0,0 +1,274 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+import httpx
+
+from ..._types import NOT_GIVEN, Body, Query, Headers, NotGiven
+from ..._compat import cached_property
+from ..._resource import SyncAPIResource, AsyncAPIResource
+from ..._response import (
+    to_raw_response_wrapper,
+    to_streamed_response_wrapper,
+    async_to_raw_response_wrapper,
+    async_to_streamed_response_wrapper,
+)
+from ..._base_client import make_request_options
+from .request.request import (
+    RequestResource,
+    AsyncRequestResource,
+    RequestResourceWithRawResponse,
+    AsyncRequestResourceWithRawResponse,
+    RequestResourceWithStreamingResponse,
+    AsyncRequestResourceWithStreamingResponse,
+)
+from ...types.workflow_results_response import WorkflowResultsResponse
+from ...types.workflow_describe_response import WorkflowDescribeResponse
+
+__all__ = ["WorkflowsResource", "AsyncWorkflowsResource"]
+
+
+class WorkflowsResource(SyncAPIResource):
+    @cached_property
+    def request(self) -> RequestResource:
+        return RequestResource(self._client)
+
+    @cached_property
+    def with_raw_response(self) -> WorkflowsResourceWithRawResponse:
+        """
+        This property can be used as a prefix for any HTTP method call to return
+        the raw response object instead of the parsed content.
+
+        For more information, see https://www.github.com/evrimai/cartography-client#accessing-raw-response-data-eg-headers
+        """
+        return WorkflowsResourceWithRawResponse(self)
+
+    @cached_property
+    def with_streaming_response(self) -> WorkflowsResourceWithStreamingResponse:
+        """
+        An alternative to `.with_raw_response` that doesn't eagerly read the response body.
+
+        For more information, see https://www.github.com/evrimai/cartography-client#with_streaming_response
+        """
+        return WorkflowsResourceWithStreamingResponse(self)
+
+    def describe(
+        self,
+        workflow_id: str,
+        *,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> WorkflowDescribeResponse:
+        """
+        Get Workflow Description
+
+        Args:
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        if not workflow_id:
+            raise ValueError(f"Expected a non-empty value for `workflow_id` but received {workflow_id!r}")
+        return self._get(
+            f"/workflows/describe/{workflow_id}",
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=WorkflowDescribeResponse,
+        )
+
+    def results(
+        self,
+        workflow_id: str,
+        *,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> WorkflowResultsResponse:
+        """
+        Get Workflow Results
+
+        Args:
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        if not workflow_id:
+            raise ValueError(f"Expected a non-empty value for `workflow_id` but received {workflow_id!r}")
+        return self._get(
+            f"/workflows/results/{workflow_id}",
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=WorkflowResultsResponse,
+        )
+
+
+class AsyncWorkflowsResource(AsyncAPIResource):
+    @cached_property
+    def request(self) -> AsyncRequestResource:
+        return AsyncRequestResource(self._client)
+
+    @cached_property
+    def with_raw_response(self) -> AsyncWorkflowsResourceWithRawResponse:
+        """
+        This property can be used as a prefix for any HTTP method call to return
+        the raw response object instead of the parsed content.
+
+        For more information, see https://www.github.com/evrimai/cartography-client#accessing-raw-response-data-eg-headers
+        """
+        return AsyncWorkflowsResourceWithRawResponse(self)
+
+    @cached_property
+    def with_streaming_response(self) -> AsyncWorkflowsResourceWithStreamingResponse:
+        """
+        An alternative to `.with_raw_response` that doesn't eagerly read the response body.
+
+        For more information, see https://www.github.com/evrimai/cartography-client#with_streaming_response
+        """
+        return AsyncWorkflowsResourceWithStreamingResponse(self)
+
+    async def describe(
+        self,
+        workflow_id: str,
+        *,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> WorkflowDescribeResponse:
+        """
+        Get Workflow Description
+
+        Args:
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        if not workflow_id:
+            raise ValueError(f"Expected a non-empty value for `workflow_id` but received {workflow_id!r}")
+        return await self._get(
+            f"/workflows/describe/{workflow_id}",
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=WorkflowDescribeResponse,
+        )
+
+    async def results(
+        self,
+        workflow_id: str,
+        *,
+        # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
+        # The extra values given here take precedence over values defined on the client or passed to this method.
+        extra_headers: Headers | None = None,
+        extra_query: Query | None = None,
+        extra_body: Body | None = None,
+        timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
+    ) -> WorkflowResultsResponse:
+        """
+        Get Workflow Results
+
+        Args:
+          extra_headers: Send extra headers
+
+          extra_query: Add additional query parameters to the request
+
+          extra_body: Add additional JSON properties to the request
+
+          timeout: Override the client-level default timeout for this request, in seconds
+        """
+        if not workflow_id:
+            raise ValueError(f"Expected a non-empty value for `workflow_id` but received {workflow_id!r}")
+        return await self._get(
+            f"/workflows/results/{workflow_id}",
+            options=make_request_options(
+                extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
+            ),
+            cast_to=WorkflowResultsResponse,
+        )
+
+
+class WorkflowsResourceWithRawResponse:
+    def __init__(self, workflows: WorkflowsResource) -> None:
+        self._workflows = workflows
+
+        self.describe = to_raw_response_wrapper(
+            workflows.describe,
+        )
+        self.results = to_raw_response_wrapper(
+            workflows.results,
+        )
+
+    @cached_property
+    def request(self) -> RequestResourceWithRawResponse:
+        return RequestResourceWithRawResponse(self._workflows.request)
+
+
+class AsyncWorkflowsResourceWithRawResponse:
+    def __init__(self, workflows: AsyncWorkflowsResource) -> None:
+        self._workflows = workflows
+
+        self.describe = async_to_raw_response_wrapper(
+            workflows.describe,
+        )
+        self.results = async_to_raw_response_wrapper(
+            workflows.results,
+        )
+
+    @cached_property
+    def request(self) -> AsyncRequestResourceWithRawResponse:
+        return AsyncRequestResourceWithRawResponse(self._workflows.request)
+
+
+class WorkflowsResourceWithStreamingResponse:
+    def __init__(self, workflows: WorkflowsResource) -> None:
+        self._workflows = workflows
+
+        self.describe = to_streamed_response_wrapper(
+            workflows.describe,
+        )
+        self.results = to_streamed_response_wrapper(
+            workflows.results,
+        )
+
+    @cached_property
+    def request(self) -> RequestResourceWithStreamingResponse:
+        return RequestResourceWithStreamingResponse(self._workflows.request)
+
+
+class AsyncWorkflowsResourceWithStreamingResponse:
+    def __init__(self, workflows: AsyncWorkflowsResource) -> None:
+        self._workflows = workflows
+
+        self.describe = async_to_streamed_response_wrapper(
+            workflows.describe,
+        )
+        self.results = async_to_streamed_response_wrapper(
+            workflows.results,
+        )
+
+    @cached_property
+    def request(self) -> AsyncRequestResourceWithStreamingResponse:
+        return AsyncRequestResourceWithStreamingResponse(self._workflows.request)

cartography/types/__init__.py

@@ -0,0 +1,23 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from .wait_until import WaitUntil as WaitUntil
+from .downloader_type import DownloaderType as DownloaderType
+from .bulk_scrape_result import BulkScrapeResult as BulkScrapeResult
+from .scrape_engine_param import ScrapeEngineParam as ScrapeEngineParam
+from .bulk_download_result import BulkDownloadResult as BulkDownloadResult
+from .health_check_response import HealthCheckResponse as HealthCheckResponse
+from .crawl_create_graph_params import CrawlCreateGraphParams as CrawlCreateGraphParams
+from .scrape_scrape_bulk_params import ScrapeScrapeBulkParams as ScrapeScrapeBulkParams
+from .workflow_results_response import WorkflowResultsResponse as WorkflowResultsResponse
+from .api_info_retrieve_response import APIInfoRetrieveResponse as APIInfoRetrieveResponse
+from .workflow_describe_response import WorkflowDescribeResponse as WorkflowDescribeResponse
+from .crawl_create_graph_response import CrawlCreateGraphResponse as CrawlCreateGraphResponse
+from .download_create_bulk_params import DownloadCreateBulkParams as DownloadCreateBulkParams
+from .scrape_scrape_bulk_response import ScrapeScrapeBulkResponse as ScrapeScrapeBulkResponse
+from .scrape_scrape_single_params import ScrapeScrapeSingleParams as ScrapeScrapeSingleParams
+from .download_create_bulk_response import DownloadCreateBulkResponse as DownloadCreateBulkResponse
+from .download_create_single_params import DownloadCreateSingleParams as DownloadCreateSingleParams
+from .scrape_scrape_single_response import ScrapeScrapeSingleResponse as ScrapeScrapeSingleResponse
+from .download_create_single_response import DownloadCreateSingleResponse as DownloadCreateSingleResponse

cartography/types/api_info_retrieve_response.py

@@ -0,0 +1,8 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict
+from typing_extensions import TypeAlias
+
+__all__ = ["APIInfoRetrieveResponse"]
+
+APIInfoRetrieveResponse: TypeAlias = Dict[str, object]

cartography/types/bulk_download_result.py

@@ -0,0 +1,23 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+from typing_extensions import Literal
+
+from .._models import BaseModel
+
+__all__ = ["BulkDownloadResult"]
+
+
+class BulkDownloadResult(BaseModel):
+    status: Literal["success", "cached", "failed", "save_failed"]
+    """Status of bulk download result"""
+
+    url: str
+
+    error: Optional[str] = None
+
+    job_id: Optional[str] = None
+
+    s3_bucket: Optional[str] = None
+
+    s3_key: Optional[str] = None

cartography/types/bulk_scrape_result.py

@@ -0,0 +1,19 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+
+from .._models import BaseModel
+
+__all__ = ["BulkScrapeResult"]
+
+
+class BulkScrapeResult(BaseModel):
+    status: str
+
+    url: str
+
+    composite_hash: Optional[str] = None
+
+    error: Optional[str] = None
+
+    status_code: Optional[int] = None

cartography/types/crawl_create_graph_params.py

@@ -0,0 +1,46 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, Iterable, Optional
+from typing_extensions import Required, TypedDict
+
+__all__ = ["CrawlCreateGraphParams"]
+
+
+class CrawlCreateGraphParams(TypedDict, total=False):
+    crawl_id: Required[str]
+    """Unique identifier for this crawl"""
+
+    engines: Required[Iterable[Dict[str, object]]]
+    """List of engines to use"""
+
+    s3_bucket: Required[str]
+    """S3 bucket for checkpointing"""
+
+    url: Required[str]
+    """Root URL to start crawling from"""
+
+    absolute_only: bool
+    """Only extract absolute URLs"""
+
+    batch_size: int
+    """URLs per batch"""
+
+    debug: bool
+    """Enable debug information"""
+
+    depth: Optional[int]
+    """Maximum crawl depth"""
+
+    keep_external: bool
+    """Keep external URLs in results"""
+
+    max_urls: int
+    """Maximum URLs to crawl"""
+
+    max_workers: int
+    """Maximum concurrent workers"""
+
+    visit_external: bool
+    """Visit external URLs"""

cartography/types/crawl_create_graph_response.py

@@ -0,0 +1,37 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+
+from .._models import BaseModel
+
+__all__ = ["CrawlCreateGraphResponse"]
+
+
+class CrawlCreateGraphResponse(BaseModel):
+    absolute_only: bool
+
+    batch_size: int
+
+    crawl_id: str
+
+    depth_reached: int
+
+    keep_external: bool
+
+    max_depth: int
+
+    max_urls: int
+
+    max_workers: int
+
+    root_url: str
+
+    timestamp: str
+
+    total_pages_visited: int
+
+    visit_external: bool
+
+    visited_urls: List[str]
+
+    debug_frame: Optional[List[Dict[str, object]]] = None

cartography/types/download_create_bulk_params.py

@@ -0,0 +1,37 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import List
+from typing_extensions import Required, TypedDict
+
+from .wait_until import WaitUntil
+from .downloader_type import DownloaderType
+
+__all__ = ["DownloadCreateBulkParams"]
+
+
+class DownloadCreateBulkParams(TypedDict, total=False):
+    crawl_id: Required[str]
+    """Unique identifier for this crawl"""
+
+    s3_bucket: Required[str]
+    """S3 bucket for storage and checkpoints"""
+
+    urls: Required[List[str]]
+    """List of URLs to download"""
+
+    batch_size: int
+    """URLs per batch"""
+
+    debug: bool
+    """Enable debug information"""
+
+    downloader_type: DownloaderType
+    """Available downloader types"""
+
+    max_workers: int
+    """Maximum concurrent workers"""
+
+    wait_until: WaitUntil
+    """When to consider downloads complete"""