garf-executors 0.2.3__py3-none-any.whl → 1.1.3__py3-none-any.whl

{garf_executors → garf/executors}/entrypoints/grpc_server.py

@@ -19,29 +19,42 @@ import logging
 from concurrent import futures
 
 import grpc
+from garf.executors import execution_context, garf_pb2, garf_pb2_grpc, setup
+from garf.executors.entrypoints.tracer import initialize_tracer
 from google.protobuf.json_format import MessageToDict
 from grpc_reflection.v1alpha import reflection
 
-import garf_executors
-from garf_executors import garf_pb2, garf_pb2_grpc
-from garf_executors.entrypoints.tracer import initialize_tracer
-
 
 class GarfService(garf_pb2_grpc.GarfService):
   def Execute(self, request, context):
-    query_executor = garf_executors.setup_executor(
+    query_executor = setup.setup_executor(
       request.source, request.context.fetcher_parameters
     )
-    execution_context = garf_executors.execution_context.ExecutionContext(
-      **MessageToDict(request.context, preserving_proto_field_name=True)
-    )
     result = query_executor.execute(
       query=request.query,
       title=request.title,
-      context=execution_context,
+      context=execution_context.ExecutionContext(
+        **MessageToDict(request.context, preserving_proto_field_name=True)
+      ),
     )
     return garf_pb2.ExecuteResponse(results=[result])
 
+  def Fetch(self, request, context):
+    query_executor = setup.setup_executor(
+      request.source, request.context.fetcher_parameters
+    )
+    query_args = execution_context.ExecutionContext(
+      **MessageToDict(request.context, preserving_proto_field_name=True)
+    ).query_parameters
+    result = query_executor.fetcher.fetch(
+      query_specification=request.query,
+      title=request.title,
+      args=query_args,
+    )
+    return garf_pb2.FetchResponse(
+      columns=result.column_names, rows=result.to_list(row_type='dict')
+    )
+
 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()

garf/executors/entrypoints/server.py

@@ -0,0 +1,174 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""FastAPI endpoint for executing queries."""
+
+from typing import Optional, Union
+
+import fastapi
+import garf.executors
+import pydantic
+import typer
+import uvicorn
+from garf.executors import exceptions, setup
+from garf.executors.entrypoints import utils
+from garf.executors.entrypoints.tracer import (
+  initialize_meter,
+  initialize_tracer,
+)
+from garf.executors.workflows import workflow_runner
+from garf.io import reader
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing_extensions import Annotated
+
+initialize_tracer()
+initialize_meter()
+app = fastapi.FastAPI()
+FastAPIInstrumentor.instrument_app(app)
+typer_app = typer.Typer()
+
+
+class GarfSettings(BaseSettings):
+  """Specifies environmental variables for garf.
+
+  Ensure that mandatory variables are exposed via
+  export ENV_VARIABLE_NAME=VALUE.
+
+  Attributes:
+    loglevel: Level of logging.
+    log_name: Name of log.
+    logger_type: Type of logger.
+  """
+
+  model_config = SettingsConfigDict(env_prefix='garf_')
+
+  loglevel: str = 'INFO'
+  log_name: str = 'garf'
+  logger_type: str = 'local'
+
+
+class GarfDependencies:
+  def __init__(self) -> None:
+    """Initializes GarfDependencies."""
+    settings = GarfSettings()
+    self.logger = utils.init_logging(
+      loglevel=settings.loglevel,
+      logger_type=settings.logger_type,
+      name=settings.log_name,
+    )
+
+
+class ApiExecutorRequest(pydantic.BaseModel):
+  """Request for executing a query.
+
+  Attributes:
+    source: Type of API to interact with.
+    title: Name of the query used as an output for writing.
+    query: Query to execute.
+    query_path: Local or remote path to query.
+    context: Execution context.
+  """
+
+  source: str
+  title: Optional[str] = None
+  query: Optional[str] = None
+  query_path: Optional[Union[str, list[str]]] = None
+  context: garf.executors.api_executor.ApiExecutionContext
+
+  @pydantic.model_validator(mode='after')
+  def check_query_specified(self):
+    if not self.query_path and not self.query:
+      raise exceptions.GarfExecutorError(
+        'Missing one of required parameters: query, query_path'
+      )
+    return self
+
+  def model_post_init(self, __context__) -> None:
+    if self.query_path and isinstance(self.query_path, str):
+      self.query = reader.FileReader().read(self.query_path)
+    if not self.title:
+      self.title = str(self.query_path)
+
+
+class ApiExecutorResponse(pydantic.BaseModel):
+  """Response after executing a query.
+
+  Attributes:
+    results: Results of query execution.
+  """
+
+  results: list[str]
+
+
+@app.get('/api/version')
+async def version() -> str:
+  return garf.executors.__version__
+
+
+@app.get('/api/fetchers')
+async def get_fetchers(
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> list[str]:
+  """Shows all available API sources."""
+  return list(garf.executors.fetchers.find_fetchers())
+
+
+@app.post('/api/execute')
+def execute(
+  request: ApiExecutorRequest,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> ApiExecutorResponse:
+  query_executor = setup.setup_executor(
+    request.source, request.context.fetcher_parameters
+  )
+  result = query_executor.execute(request.query, request.title, request.context)
+  return ApiExecutorResponse(results=[result])
+
+
+@app.post('/api/execute:batch')
+def execute_batch(
+  request: ApiExecutorRequest,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+) -> ApiExecutorResponse:
+  query_executor = setup.setup_executor(
+    request.source, request.context.fetcher_parameters
+  )
+  reader_client = reader.FileReader()
+  batch = {query: reader_client.read(query) for query in request.query_path}
+  results = query_executor.execute_batch(batch, request.context)
+  return ApiExecutorResponse(results=results)
+
+
+@app.post('/api/execute:workflow')
+def execute_workflow(
+  workflow_file: str,
+  dependencies: Annotated[GarfDependencies, fastapi.Depends(GarfDependencies)],
+  enable_cache: bool = False,
+  cache_ttl_seconds: int = 3600,
+) -> list[str]:
+  return workflow_runner.WorkflowRunner.from_file(workflow_file).run(
+    enable_cache=enable_cache, cache_ttl_seconds=cache_ttl_seconds
+  )
+
+
+@typer_app.command()
+def main(
+  port: Annotated[int, typer.Option(help='Port to start the server')] = 8000,
+):
+  uvicorn.run(app, port=port)
+
+
+if __name__ == '__main__':
+  typer_app()

garf/executors/entrypoints/tracer.py

@@ -0,0 +1,82 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from opentelemetry import metrics, trace
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+  OTLPMetricExporter,
+)
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+  OTLPSpanExporter,
+)
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+  PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+DEFAULT_SERVICE_NAME = 'garf'
+
+
+def initialize_tracer():
+  resource = Resource.create(
+    {'service.name': os.getenv('OTLP_SERVICE_NAME', DEFAULT_SERVICE_NAME)}
+  )
+
+  tracer_provider = TracerProvider(resource=resource)
+
+  if otel_endpoint := os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
+    if gcp_project_id := os.getenv('OTEL_EXPORTER_GCP_PROJECT_ID'):
+      try:
+        from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter
+      except ImportError as e:
+        raise ImportError(
+          'Please install garf-executors with GCP support '
+          '- `pip install garf-executors[gcp]`'
+        ) from e
+
+      cloud_span_processor = BatchSpanProcessor(
+        CloudTraceSpanExporter(project_id=gcp_project_id)
+      )
+      tracer_provider.add_span_processor(cloud_span_processor)
+    else:
+      otlp_processor = BatchSpanProcessor(
+        OTLPSpanExporter(endpoint=otel_endpoint, insecure=True)
+      )
+      tracer_provider.add_span_processor(otlp_processor)
+
+  trace.set_tracer_provider(tracer_provider)
+
+
+def initialize_meter():
+  resource = Resource.create(
+    {'service.name': os.getenv('OTLP_SERVICE_NAME', DEFAULT_SERVICE_NAME)}
+  )
+  meter_provider = MeterProvider(resource=resource)
+
+  if otel_endpoint := os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
+    otlp_metric_exporter = OTLPMetricExporter(
+      endpoint=f'{otel_endpoint}/v1/metrics'
+    )
+    metric_reader = PeriodicExportingMetricReader(otlp_metric_exporter)
+    meter_provider = MeterProvider(
+      resource=resource, metric_readers=[metric_reader]
+    )
+  else:
+    meter_provider = MeterProvider(resource=resource)
+  metrics.set_meter_provider(meter_provider)
+  return meter_provider

garf/executors/entrypoints/utils.py

@@ -0,0 +1,140 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for various helpers for executing Garf as CLI tool."""
+
+from __future__ import annotations
+
+import enum
+import logging
+import sys
+from collections.abc import Sequence
+from typing import Any
+
+from rich import logging as rich_logging
+
+
+class ParamsParser:
+  def __init__(self, identifiers: Sequence[str]) -> None:
+    self.identifiers = identifiers
+
+  def parse(self, params: Sequence) -> dict[str, dict | None]:
+    return {
+      identifier: self._parse_params(identifier, params)
+      for identifier in self.identifiers
+    }
+
+  def _parse_params(self, identifier: str, params: Sequence[Any]) -> dict:
+    parsed_params = {}
+    if params:
+      raw_params = [param.split('=', maxsplit=1) for param in params]
+      for param in raw_params:
+        param_pair = self._identify_param_pair(identifier, param)
+        if param_pair:
+          parsed_params.update(param_pair)
+    return parsed_params
+
+  def _identify_param_pair(
+    self, identifier: str, param: Sequence[str]
+  ) -> dict[str, Any] | None:
+    key = param[0]
+    if not identifier or identifier not in key:
+      return None
+    provided_identifier, *keys = key.split('.')
+    if not keys:
+      return None
+    if len(keys) > 1:
+      raise GarfParamsException(
+        f'{key} is invalid format,'
+        f'`--{identifier}.key=value` or `--{identifier}.key` '
+        'are the correct formats'
+      )
+    provided_identifier = provided_identifier.replace('--', '')
+    if provided_identifier not in self.identifiers:
+      supported_arguments = ', '.join(self.identifiers)
+      raise GarfParamsException(
+        f'CLI argument {provided_identifier} is not supported'
+        f', supported arguments {supported_arguments}'
+      )
+    if provided_identifier != identifier:
+      return None
+    key = keys[0].replace('-', '_')
+    if not key:
+      raise GarfParamsException(
+        f'{identifier} {key} is invalid,'
+        f'`--{identifier}.key=value` or `--{identifier}.key` '
+        'are the correct formats'
+      )
+    if len(param) == 2:
+      return {key: param[1]}
+    if len(param) == 1:
+      return {key: True}
+    raise GarfParamsException(
+      f'{identifier} {key} is invalid,'
+      f'`--{identifier}.key=value` or `--{identifier}.key` '
+      'are the correct formats'
+    )
+
+
+class GarfParamsException(Exception):
+  """Defines exception for incorrect parameters."""
+
+
+class LoggerEnum(str, enum.Enum):
+  local = 'local'
+  rich = 'rich'
+  gcloud = 'gcloud'
+
+
+def init_logging(
+  loglevel: str = 'INFO',
+  logger_type: str | LoggerEnum = 'local',
+  name: str = __name__,
+) -> logging.Logger:
+  loglevel = getattr(logging, loglevel)
+  if logger_type == 'rich':
+    logging.basicConfig(
+      format='%(message)s',
+      level=loglevel,
+      datefmt='%Y-%m-%d %H:%M:%S',
+      handlers=[
+        rich_logging.RichHandler(rich_tracebacks=True),
+      ],
+    )
+  elif logger_type == 'gcloud':
+    try:
+      import google.cloud.logging as glogging
+    except ImportError as e:
+      raise ImportError(
+        'Please install garf-executors with Cloud logging support - '
+        '`pip install garf-executors[bq]`'
+      ) from e
+
+    client = glogging.Client()
+    handler = glogging.handlers.CloudLoggingHandler(client, name=name)
+    handler.close()
+    glogging.handlers.setup_logging(handler, log_level=loglevel)
+    logging.basicConfig(
+      level=loglevel,
+      handlers=[handler],
+    )
+  else:
+    logging.basicConfig(
+      format='[%(asctime)s][%(name)s][%(levelname)s] %(message)s',
+      stream=sys.stdout,
+      level=loglevel,
+      datefmt='%Y-%m-%d %H:%M:%S',
+    )
+  logging.getLogger('smart_open.smart_open_lib').setLevel(logging.WARNING)
+  logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
+  return logging.getLogger(name)

garf/executors/exceptions.py

@@ -0,0 +1,17 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class GarfExecutorError(Exception):
+  """Base class for garf executor exceptions."""

garf/executors/execution_context.py

@@ -0,0 +1,117 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=C0330, g-bad-import-order, g-multiple-import
+
+"""Captures parameters for fetching data from APIs."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+from typing import Any
+
+import pydantic
+import smart_open
+import yaml
+from garf.core import query_editor
+from garf.io import writer
+from garf.io.writers import abs_writer
+
+
+class ExecutionContext(pydantic.BaseModel):
+  """Common context for executing one or more queries.
+
+  Attributes:
+    query_parameters: Parameters to dynamically change query text.
+    fetcher_parameters: Parameters to specify fetching setup.
+    writer: Type of writer to use. Can be a single writer string or list of writers.
+    writer_parameters: Optional parameters to setup writer.
+  """
+
+  query_parameters: query_editor.GarfQueryParameters | None = pydantic.Field(
+    default_factory=dict
+  )
+  fetcher_parameters: dict[str, Any] | None = pydantic.Field(
+    default_factory=dict
+  )
+  writer: str | list[str] | None = None
+  writer_parameters: dict[str, str] | None = pydantic.Field(
+    default_factory=dict
+  )
+
+  def model_post_init(self, __context__) -> None:
+    if self.fetcher_parameters is None:
+      self.fetcher_parameters = {}
+    if self.writer_parameters is None:
+      self.writer_parameters = {}
+    if not self.query_parameters:
+      self.query_parameters = query_editor.GarfQueryParameters()
+
+  @classmethod
+  def from_file(
+    cls, path: str | pathlib.Path | os.PathLike[str]
+  ) -> ExecutionContext:
+    """Builds context from local or remote yaml file."""
+    with smart_open.open(path, 'r', encoding='utf-8') as f:
+      data = yaml.safe_load(f)
+    return ExecutionContext(**data)
+
+  def save(self, path: str | pathlib.Path | os.PathLike[str]) -> str:
+    """Saves context to local or remote yaml file."""
+    with smart_open.open(path, 'w', encoding='utf-8') as f:
+      yaml.dump(self.model_dump(), f, encoding='utf-8')
+    return f'ExecutionContext is saved to {str(path)}'
+
+  @property
+  def writer_client(self) -> abs_writer.AbsWriter:
+    """Returns single writer client."""
+    if isinstance(self.writer, list) and len(self.writer) > 0:
+      writer_type = self.writer[0]
+    else:
+      writer_type = self.writer
+
+    writer_params = self.writer_parameters or {}
+
+    if not writer_type:
+      raise ValueError('No writer specified')
+
+    writer_client = writer.create_writer(writer_type, **writer_params)
+    if writer_type == 'bq':
+      _ = writer_client.create_or_get_dataset()
+    if writer_type == 'sheet':
+      writer_client.init_client()
+    return writer_client
+
+  @property
+  def writer_clients(self) -> list[abs_writer.AbsWriter]:
+    """Returns list of writer clients."""
+    if not self.writer:
+      return []
+
+    # Convert single writer to list for uniform processing
+    writers_to_use = (
+      self.writer if isinstance(self.writer, list) else [self.writer]
+    )
+    writer_params = self.writer_parameters or {}
+
+    clients = []
+    for writer_type in writers_to_use:
+      writer_client = writer.create_writer(writer_type, **writer_params)
+      if writer_type == 'bq':
+        _ = writer_client.create_or_get_dataset()
+      if writer_type == 'sheet':
+        writer_client.init_client()
+      clients.append(writer_client)
+    return clients