graphdatascience 1.16__tar.gz → 1.17__tar.gz

{graphdatascience-1.16/graphdatascience.egg-info → graphdatascience-1.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: graphdatascience
-Version: 1.16
+Version: 1.17
 Summary: A Python client for the Neo4j Graph Data Science (GDS) library
 Home-page: https://neo4j.com/product/graph-data-science/
 Author: Neo4j
@@ -38,6 +38,7 @@ Requires-Dist: tqdm<5.0,>=4.0
 Requires-Dist: typing-extensions<5.0,>=4.0
 Requires-Dist: requests
 Requires-Dist: tenacity>=9.0
+Requires-Dist: pydantic>=2.11
 Provides-Extra: ogb
 Requires-Dist: ogb<2.0,>=1.0; extra == "ogb"
 Provides-Extra: networkx

graphdatascience-1.17/graphdatascience/arrow_client/arrow_base_model.py

@@ -0,0 +1,12 @@
+from typing import Any
+
+from pydantic import BaseModel
+from pydantic.alias_generators import to_camel
+
+
+class ArrowBaseModel(BaseModel, alias_generator=to_camel):
+    def dump_camel(self) -> dict[str, Any]:
+        return self.model_dump(by_alias=True)
+
+    def dump_json(self) -> str:
+        return self.model_dump_json(by_alias=True)

graphdatascience-1.17/graphdatascience/arrow_client/authenticated_flight_client.py

@@ -0,0 +1,219 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from typing import Any, Iterator, Optional, Tuple, Union
+
+from pyarrow import __version__ as arrow_version
+from pyarrow import flight
+from pyarrow._flight import (
+    Action,
+    FlightInternalError,
+    FlightStreamReader,
+    FlightTimedOutError,
+    FlightUnavailableError,
+    Result,
+    Ticket,
+)
+from tenacity import retry, retry_any, retry_if_exception_type, stop_after_attempt, stop_after_delay, wait_exponential
+
+from graphdatascience.arrow_client.arrow_authentication import ArrowAuthentication
+from graphdatascience.arrow_client.arrow_info import ArrowInfo
+from graphdatascience.retry_utils.retry_config import RetryConfig
+
+from ..retry_utils.retry_utils import before_log
+from ..version import __version__
+from .middleware.auth_middleware import AuthFactory, AuthMiddleware
+from .middleware.user_agent_middleware import UserAgentFactory
+
+
+class AuthenticatedArrowClient:
+    @staticmethod
+    def create(
+        arrow_info: ArrowInfo,
+        auth: Optional[ArrowAuthentication] = None,
+        encrypted: bool = False,
+        arrow_client_options: Optional[dict[str, Any]] = None,
+        connection_string_override: Optional[str] = None,
+        retry_config: Optional[RetryConfig] = None,
+        advertised_listen_address: Optional[Tuple[str, int]] = None,
+    ) -> AuthenticatedArrowClient:
+        connection_string: str
+        if connection_string_override is not None:
+            connection_string = connection_string_override
+        else:
+            connection_string = arrow_info.listenAddress
+
+        host, port = connection_string.split(":")
+
+        if retry_config is None:
+            retry_config = RetryConfig(
+                retry=retry_any(
+                    retry_if_exception_type(FlightTimedOutError),
+                    retry_if_exception_type(FlightUnavailableError),
+                    retry_if_exception_type(FlightInternalError),
+                ),
+                stop=(stop_after_delay(10) | stop_after_attempt(5)),
+                wait=wait_exponential(multiplier=1, min=1, max=10),
+            )
+
+        return AuthenticatedArrowClient(
+            host=host,
+            retry_config=retry_config,
+            port=int(port),
+            auth=auth,
+            encrypted=encrypted,
+            arrow_client_options=arrow_client_options,
+            advertised_listen_address=advertised_listen_address,
+        )
+
+    def __init__(
+        self,
+        host: str,
+        retry_config: RetryConfig,
+        port: int = 8491,
+        auth: Optional[ArrowAuthentication] = None,
+        encrypted: bool = False,
+        arrow_client_options: Optional[dict[str, Any]] = None,
+        user_agent: Optional[str] = None,
+        advertised_listen_address: Optional[Tuple[str, int]] = None,
+    ):
+        """Creates a new GdsArrowClient instance.
+
+        Parameters
+        ----------
+        host: str
+            The host address of the GDS Arrow server
+        port: int
+            The host port of the GDS Arrow server (default is 8491)
+        auth: Optional[ArrowAuthentication]
+            An implementation of ArrowAuthentication providing a pair to be used for basic authentication
+        encrypted: bool
+            A flag that indicates whether the connection should be encrypted (default is False)
+        arrow_client_options: Optional[dict[str, Any]]
+            Additional options to be passed to the Arrow Flight client.
+        user_agent: Optional[str]
+            The user agent string to use for the connection. (default is `neo4j-graphdatascience-v[VERSION] pyarrow-v[PYARROW_VERSION])
+        retry_config: Optional[RetryConfig]
+            The retry configuration to use for the Arrow requests send by the client.
+        advertised_listen_address: Optional[Tuple[str, int]]
+            The advertised listen address of the GDS Arrow server. This will be used by remote projection and writeback operations.
+        """
+        self._host = host
+        self._port = port
+        self._auth = None
+        self._encrypted = encrypted
+        self._arrow_client_options = arrow_client_options
+        self._user_agent = user_agent
+        self._retry_config = retry_config
+        self._logger = logging.getLogger("gds_arrow_client")
+        self._retry_config = retry_config
+        if auth:
+            self._auth = auth
+            self._auth_middleware = AuthMiddleware(auth)
+        self.advertised_listen_address = advertised_listen_address
+
+        self._flight_client = self._instantiate_flight_client()
+
+    def connection_info(self) -> ConnectionInfo:
+        """
+        Returns the host and port of the GDS Arrow server.
+
+        Returns
+        -------
+        tuple[str, int]
+            the host and port of the GDS Arrow server
+        """
+        return ConnectionInfo(self._host, self._port, self._encrypted)
+
+    def advertised_connection_info(self) -> ConnectionInfo:
+        """
+        Returns the advertised host and port of the GDS Arrow server.
+
+        Returns
+        -------
+        ConnectionInfo
+            the host and port of the GDS Arrow server
+        """
+        if self.advertised_listen_address is None:
+            return self.connection_info()
+
+        h, p = self.advertised_listen_address
+        return ConnectionInfo(h, p, self._encrypted)
+
+    def request_token(self) -> Optional[str]:
+        """
+        Requests a token from the server and returns it.
+
+        Returns
+        -------
+        Optional[str]
+            a token from the server and returns it.
+        """
+
+        @retry(
+            reraise=True,
+            before=before_log("Request token", self._logger, logging.DEBUG),
+            retry=self._retry_config.retry,
+            stop=self._retry_config.stop,
+            wait=self._retry_config.wait,
+        )
+        def auth_with_retry() -> None:
+            client = self._flight_client
+            if self._auth:
+                auth_pair = self._auth.auth_pair()
+                client.authenticate_basic_token(auth_pair[0], auth_pair[1])
+
+        if self._auth:
+            auth_with_retry()
+            return self._auth_middleware.token()
+        else:
+            return "IGNORED"
+
+    def get_stream(self, ticket: Ticket) -> FlightStreamReader:
+        return self._flight_client.do_get(ticket)
+
+    def do_action(self, endpoint: str, payload: Union[bytes, dict[str, Any]]) -> Iterator[Result]:
+        payload_bytes = payload if isinstance(payload, bytes) else json.dumps(payload).encode("utf-8")
+
+        return self._flight_client.do_action(Action(endpoint, payload_bytes))  # type: ignore
+
+    def do_action_with_retry(self, endpoint: str, payload: Union[bytes, dict[str, Any]]) -> Iterator[Result]:
+        @retry(
+            reraise=True,
+            before=before_log("Send action", self._logger, logging.DEBUG),
+            retry=self._retry_config.retry,
+            stop=self._retry_config.stop,
+            wait=self._retry_config.wait,
+        )
+        def run_with_retry() -> Iterator[Result]:
+            return self.do_action(endpoint, payload)
+
+        return run_with_retry()
+
+    def _instantiate_flight_client(self) -> flight.FlightClient:
+        location = (
+            flight.Location.for_grpc_tls(self._host, self._port)
+            if self._encrypted
+            else flight.Location.for_grpc_tcp(self._host, self._port)
+        )
+        client_options: dict[str, Any] = (self._arrow_client_options or {}).copy()
+        if self._auth:
+            user_agent = f"neo4j-graphdatascience-v{__version__} pyarrow-v{arrow_version}"
+            if self._user_agent:
+                user_agent = self._user_agent
+
+            client_options["middleware"] = [
+                AuthFactory(self._auth_middleware),
+                UserAgentFactory(useragent=user_agent),
+            ]
+
+        return flight.FlightClient(location, **client_options)
+
+
+@dataclass
+class ConnectionInfo:
+    host: str
+    port: int
+    encrypted: bool

graphdatascience-1.17/graphdatascience/arrow_client/middleware/auth_middleware.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import base64
+import time
+from typing import Any, Optional
+
+from pyarrow._flight import ClientMiddleware, ClientMiddlewareFactory
+
+from graphdatascience.arrow_client.arrow_authentication import ArrowAuthentication
+
+
+class AuthFactory(ClientMiddlewareFactory):  # type: ignore
+    def __init__(self, middleware: AuthMiddleware, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._middleware = middleware
+
+    def start_call(self, info: Any) -> AuthMiddleware:
+        return self._middleware
+
+
+class AuthMiddleware(ClientMiddleware):  # type: ignore
+    def __init__(self, auth: ArrowAuthentication, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._auth = auth
+        self._token: Optional[str] = None
+        self._token_timestamp = 0
+
+    def token(self) -> Optional[str]:
+        # check whether the token is older than 10 minutes. If so, reset it.
+        if self._token and int(time.time()) - self._token_timestamp > 600:
+            self._token = None
+
+        return self._token
+
+    def _set_token(self, token: str) -> None:
+        self._token = token
+        self._token_timestamp = int(time.time())
+
+    def received_headers(self, headers: dict[str, Any]) -> None:
+        auth_header = headers.get("authorization", None)
+        if not auth_header:
+            return
+
+        # the result is always a list
+        header_value = auth_header[0]
+
+        if not isinstance(header_value, str):
+            raise ValueError(f"Incompatible header value received from server: `{header_value}`")
+
+        auth_type, token = header_value.split(" ", 1)
+        if auth_type == "Bearer":
+            self._set_token(token)
+
+    def sending_headers(self) -> dict[str, str]:
+        token = self.token()
+        if token is not None:
+            return {"authorization": "Bearer " + token}
+
+        auth_pair = self._auth.auth_pair()
+        auth_token = f"{auth_pair[0]}:{auth_pair[1]}"
+        auth_token = "Basic " + base64.b64encode(auth_token.encode("utf-8")).decode("ASCII")
+        # There seems to be a bug, `authorization` must be lower key
+        return {"authorization": auth_token}

graphdatascience-1.17/graphdatascience/arrow_client/middleware/user_agent_middleware.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pyarrow._flight import ClientMiddleware, ClientMiddlewareFactory
+
+
+class UserAgentFactory(ClientMiddlewareFactory):  # type: ignore
+    def __init__(self, useragent: str, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._middleware = UserAgentMiddleware(useragent)
+
+    def start_call(self, info: Any) -> ClientMiddleware:
+        return self._middleware
+
+
+class UserAgentMiddleware(ClientMiddleware):  # type: ignore
+    def __init__(self, useragent: str, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._useragent = useragent
+
+    def sending_headers(self) -> dict[str, str]:
+        return {"x-gds-user-agent": self._useragent}
+
+    def received_headers(self, headers: dict[str, Any]) -> None:
+        pass

graphdatascience-1.17/graphdatascience/arrow_client/v2/api_types.py

@@ -0,0 +1,17 @@
+from graphdatascience.arrow_client.arrow_base_model import ArrowBaseModel
+
+
+class JobIdConfig(ArrowBaseModel):
+    job_id: str
+
+
+class JobStatus(ArrowBaseModel):
+    job_id: str
+    status: str
+    progress: float
+
+
+class MutateResult(ArrowBaseModel):
+    mutate_millis: int
+    node_properties_written: int
+    relationships_written: int

graphdatascience-1.17/graphdatascience/arrow_client/v2/data_mapper_utils.py

@@ -0,0 +1,19 @@
+import json
+from typing import Any, Iterator
+
+from pyarrow._flight import Result
+
+
+def deserialize_single(input_stream: Iterator[Result]) -> dict[str, Any]:
+    rows = deserialize(input_stream)
+    if len(rows) != 1:
+        raise ValueError(f"Expected exactly one result, got {len(rows)}")
+
+    return rows[0]
+
+
+def deserialize(input_stream: Iterator[Result]) -> list[dict[str, Any]]:
+    def deserialize_row(row: Result):  # type:ignore
+        return json.loads(row.body.to_pybytes().decode())
+
+    return [deserialize_row(row) for row in list(input_stream)]

graphdatascience-1.17/graphdatascience/arrow_client/v2/job_client.py

@@ -0,0 +1,58 @@
+import json
+from typing import Any
+
+from pandas import ArrowDtype, DataFrame
+from pyarrow._flight import Ticket
+
+from graphdatascience.arrow_client.authenticated_flight_client import AuthenticatedArrowClient
+from graphdatascience.arrow_client.v2.api_types import JobIdConfig, JobStatus
+from graphdatascience.arrow_client.v2.data_mapper_utils import deserialize_single
+
+JOB_STATUS_ENDPOINT = "v2/jobs.status"
+RESULTS_SUMMARY_ENDPOINT = "v2/results.summary"
+
+
+class JobClient:
+    @staticmethod
+    def run_job_and_wait(client: AuthenticatedArrowClient, endpoint: str, config: dict[str, Any]) -> str:
+        job_id = JobClient.run_job(client, endpoint, config)
+        JobClient.wait_for_job(client, job_id)
+        return job_id
+
+    @staticmethod
+    def run_job(client: AuthenticatedArrowClient, endpoint: str, config: dict[str, Any]) -> str:
+        res = client.do_action_with_retry(endpoint, config)
+
+        single = deserialize_single(res)
+        return JobIdConfig(**single).job_id
+
+    @staticmethod
+    def wait_for_job(client: AuthenticatedArrowClient, job_id: str) -> None:
+        while True:
+            arrow_res = client.do_action_with_retry(JOB_STATUS_ENDPOINT, JobIdConfig(jobId=job_id).dump_camel())
+            job_status = JobStatus(**deserialize_single(arrow_res))
+            if job_status.status == "Done":
+                break
+
+    @staticmethod
+    def get_summary(client: AuthenticatedArrowClient, job_id: str) -> dict[str, Any]:
+        res = client.do_action_with_retry(RESULTS_SUMMARY_ENDPOINT, JobIdConfig(jobId=job_id).dump_camel())
+        return deserialize_single(res)
+
+    @staticmethod
+    def stream_results(client: AuthenticatedArrowClient, graph_name: str, job_id: str) -> DataFrame:
+        payload = {
+            "graphName": graph_name,
+            "jobId": job_id,
+        }
+
+        res = client.do_action_with_retry("v2/results.stream", payload)
+        export_job_id = JobIdConfig(**deserialize_single(res)).job_id
+
+        stream_payload = {"version": "v2", "name": export_job_id, "body": {}}
+
+        ticket = Ticket(json.dumps(stream_payload).encode("utf-8"))
+
+        get = client.get_stream(ticket)
+        arrow_table = get.read_all()
+        return arrow_table.to_pandas(types_mapper=ArrowDtype)  # type: ignore

graphdatascience-1.17/graphdatascience/arrow_client/v2/mutation_client.py

@@ -0,0 +1,18 @@
+import math
+import time
+
+from graphdatascience.arrow_client.authenticated_flight_client import AuthenticatedArrowClient
+from graphdatascience.arrow_client.v2.api_types import MutateResult
+from graphdatascience.arrow_client.v2.data_mapper_utils import deserialize_single
+
+
+class MutationClient:
+    MUTATE_ENDPOINT = "v2/results.mutate"
+
+    @staticmethod
+    def mutate_node_property(client: AuthenticatedArrowClient, job_id: str, mutate_property: str) -> MutateResult:
+        mutate_config = {"jobId": job_id, "mutateProperty": mutate_property}
+        start_time = time.time()
+        mutate_arrow_res = client.do_action_with_retry(MutationClient.MUTATE_ENDPOINT, mutate_config)
+        mutate_millis = math.ceil((time.time() - start_time) * 1000)
+        return MutateResult(mutateMillis=mutate_millis, **deserialize_single(mutate_arrow_res))

graphdatascience-1.17/graphdatascience/arrow_client/v2/remote_write_back_client.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import time
+from typing import Any, Optional
+
+from graphdatascience import QueryRunner
+from graphdatascience.arrow_client.authenticated_flight_client import AuthenticatedArrowClient
+from graphdatascience.call_parameters import CallParameters
+from graphdatascience.procedure_surface.api.base_result import BaseResult
+from graphdatascience.query_runner.protocol.write_protocols import WriteProtocol
+from graphdatascience.query_runner.termination_flag import TerminationFlagNoop
+from graphdatascience.session.dbms.protocol_resolver import ProtocolVersionResolver
+
+
+class RemoteWriteBackClient:
+    def __init__(self, arrow_client: AuthenticatedArrowClient, query_runner: QueryRunner):
+        self._arrow_client = arrow_client
+        self._query_runner = query_runner
+
+        protocol_version = ProtocolVersionResolver(query_runner).resolve()
+        self._write_protocol = WriteProtocol.select(protocol_version)
+
+    # TODO: Add progress logging
+    def write(
+        self,
+        graph_name: str,
+        job_id: str,
+        concurrency: Optional[int] = None,
+        property_overwrites: Optional[dict[str, str]] = None,
+        relationship_type_overwrite: Optional[str] = None,
+    ) -> WriteBackResult:
+        arrow_config = self._arrow_configuration()
+
+        configuration: dict[str, Any] = {}
+        if concurrency is not None:
+            configuration["concurrency"] = concurrency
+        if property_overwrites is not None:
+            configuration["writeProperties"] = property_overwrites
+        if relationship_type_overwrite is not None:
+            configuration["writeRelationshipType"] = relationship_type_overwrite
+
+        write_back_params = CallParameters(
+            graphName=graph_name,
+            jobId=job_id,
+            arrowConfiguration=arrow_config,
+            configuration=configuration,
+        )
+
+        start_time = time.time()
+
+        result = self._write_protocol.run_write_back(
+            self._query_runner, write_back_params, None, TerminationFlagNoop()
+        ).squeeze()
+        write_millis = int((time.time() - start_time) * 1000)
+
+        return WriteBackResult(writeMillis=write_millis, **result.squeeze())
+
+    def _arrow_configuration(self) -> dict[str, Any]:
+        connection_info = self._arrow_client.advertised_connection_info()
+        token = self._arrow_client.request_token()
+        if token is None:
+            token = "IGNORED"
+        arrow_config = {
+            "host": connection_info.host,
+            "port": connection_info.port,
+            "token": token,
+            "encrypted": connection_info.encrypted,
+        }
+
+        return arrow_config
+
+
+class WriteBackResult(BaseResult):
+    written_node_properties: int
+    written_node_labels: int
+    written_relationships: int
+    write_millis: int
+    status: str
+    progress: float

{graphdatascience-1.16 → graphdatascience-1.17}/graphdatascience/graph/graph_cypher_runner.py

@@ -6,6 +6,8 @@ from typing import Any, Optional
 
 from pandas import Series
 
+from graphdatascience.query_runner.query_mode import QueryMode
+
 from ..caller_base import CallerBase
 from ..query_runner.query_runner import QueryRunner
 from ..server_version.server_version import ServerVersion
@@ -46,7 +48,7 @@ class GraphCypherRunner(CallerBase):
         GraphCypherRunner._verify_query_ends_with_return_clause(self._namespace, query)
 
         result: Optional[dict[str, Any]] = self._query_runner.run_retryable_cypher(
-            query, params, database, custom_error=False
+            query, params, database, custom_error=False, mode=QueryMode.READ
         ).squeeze()
 
         if not result:

{graphdatascience-1.16 → graphdatascience-1.17}/graphdatascience/graph/graph_entity_ops_runner.py

@@ -5,6 +5,8 @@ from warnings import filterwarnings
 import pandas as pd
 from pandas import DataFrame, Series
 
+from graphdatascience.query_runner.query_mode import QueryMode
+
 from ..call_parameters import CallParameters
 from ..error.cypher_warning_handler import (
     filter_id_func_deprecation_warning,
@@ -164,7 +166,9 @@ class GraphNodePropertiesRunner(GraphEntityOpsBaseRunner):
             unique_node_ids = result["nodeId"].drop_duplicates().tolist()
 
             db_properties_df = query_runner.run_retryable_cypher(
-                GraphNodePropertiesRunner._build_query(db_node_properties), params={"ids": unique_node_ids}
+                GraphNodePropertiesRunner._build_query(db_node_properties),
+                params={"ids": unique_node_ids},
+                mode=QueryMode.READ,
             )
 
             if "propertyValue" not in result.keys():

{graphdatascience-1.16 → graphdatascience-1.17}/graphdatascience/graph/graph_object.py

@@ -68,7 +68,7 @@ class Graph:
         """
         return self._graph_info(["database"])  # type: ignore
 
-    def configuration(self) -> "Series[Any]":
+    def configuration(self) -> Series[Any]:
         """
         Returns:
             the configuration of the graph