garf-executors 0.0.6__py3-none-any.whl → 0.1.4__py3-none-any.whl

garf_executors/entrypoints/tracer.py

@@ -0,0 +1,42 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+  OTLPSpanExporter,
+)
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+  BatchSpanProcessor,
+)
+
+DEFAULT_SERVICE_NAME = 'garf'
+
+
+def initialize_tracer():
+  resource = Resource.create(
+    {'service.name': os.getenv('OTLP_SERVICE_NAME', DEFAULT_SERVICE_NAME)}
+  )
+
+  tracer_provider = TracerProvider(resource=resource)
+
+  if otel_endpoint := os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT'):
+    otlp_processor = BatchSpanProcessor(
+      OTLPSpanExporter(endpoint=otel_endpoint, insecure=True)
+    )
+    tracer_provider.add_span_processor(otlp_processor)
+  trace.set_tracer_provider(tracer_provider)

garf_executors/entrypoints/utils.py

@@ -15,261 +15,14 @@
 
 from __future__ import annotations
 
-import dataclasses
-import datetime
+import enum
 import logging
-import os
 import sys
-from collections.abc import MutableSequence, Sequence
-from typing import Any, TypedDict
+from collections.abc import Sequence
+from typing import Any
 
-import smart_open
-import yaml
-from dateutil import relativedelta
 from rich import logging as rich_logging
 
-from garf_core import query_editor
-
-
-class GarfQueryParameters(TypedDict):
-  """Annotation for dictionary of query specific parameters passed via CLI.
-
-  Attributes:
-      macros: Mapping for elements that will be replaced in the queries.
-      template: Mapping for elements that will rendered via Jinja templates.
-  """
-
-  macros: dict[str, str]
-  template: dict[str, str]
-
-
-@dataclasses.dataclass
-class BaseConfig:
-  """Base config to inherit other configs from."""
-
-  def __add__(self, other: BaseConfig) -> BaseConfig:
-    """Creates new config of the same type from two configs.
-
-    Parameters from added config overwrite already present parameters.
-
-    Args:
-        other: Config that could be merged with the original one.
-
-    Returns:
-        New config with values from both configs.
-    """
-    right_dict = _remove_empty_values(self.__dict__)
-    left_dict = _remove_empty_values(other.__dict__)
-    new_dict = {**right_dict, **left_dict}
-    return self.__class__(**new_dict)
-
-  @classmethod
-  def from_dict(
-    cls, config_parameters: dict[str, str | GarfQueryParameters]
-  ) -> BaseConfig:
-    """Builds config from provided parameters ignoring empty ones."""
-    return cls(**_remove_empty_values(config_parameters))
-
-
-@dataclasses.dataclass
-class GarfConfig(BaseConfig):
-  """Stores values to run garf from command line.
-
-  Attributes:
-      account:
-          Account(s) to get data from.
-      output:
-          Specifies where to store fetched data (console, csv, BQ.)
-      api_version:
-          Google Ads API version.
-      params:
-          Any parameters passed to Garf query for substitution.
-      writer_params:
-          Any parameters that can be passed to writer for data saving.
-      customer_ids_query:
-          Query text to limit accounts fetched from Ads API.
-      customer_ids_query_file:
-          Path to query to limit accounts fetched from Ads API.
-  """
-
-  account: str | list[str] | None = None
-  output: str = 'console'
-  params: GarfQueryParameters = dataclasses.field(default_factory=dict)
-  writer_params: dict[str, str | int] = dataclasses.field(default_factory=dict)
-  customer_ids_query: str | None = None
-  customer_ids_query_file: str | None = None
-
-  def __post_init__(self) -> None:
-    """Ensures that values passed during __init__ correctly formatted."""
-    if isinstance(self.account, MutableSequence):
-      self.account = [
-        str(account).replace('-', '').strip() for account in self.account
-      ]
-    else:
-      self.account = (
-        str(self.account).replace('-', '').strip() if self.account else None
-      )
-    self.writer_params = {
-      key.replace('-', '_'): value for key, value in self.writer_params.items()
-    }
-
-
-class GarfConfigException(Exception):
-  """Exception for invalid GarfConfig."""
-
-
-@dataclasses.dataclass
-class GarfBqConfig(BaseConfig):
-  """Stores values to run garf-bq from command line.
-
-  Attributes:
-      project:
-         Google Cloud project name.
-      dataset_location:
-          Location of BigQuery dataset.
-      params:
-          Any parameters passed to BigQuery query for substitution.
-  """
-
-  project: str | None = None
-  dataset_location: str | None = None
-  params: GarfQueryParameters = dataclasses.field(default_factory=dict)
-
-
-@dataclasses.dataclass
-class GarfSqlConfig(BaseConfig):
-  """Stores values to run garf-sql from command line.
-
-  Attributes:
-      connection_string:
-         Connection string to SqlAlchemy database engine.
-      params:
-          Any parameters passed to SQL query for substitution.
-  """
-
-  connection_string: str | None = None
-  params: GarfQueryParameters = dataclasses.field(default_factory=dict)
-
-
-class ConfigBuilder:
-  """Builds config of provided type.
-
-  Config can be created from file, build from arguments or both.
-
-  Attributes:
-      config: Concrete config class that needs to be built.
-  """
-
-  _config_mapping: dict[str, BaseConfig] = {
-    'garf': GarfConfig,
-    'garf-bq': GarfBqConfig,
-    'garf-sql': GarfSqlConfig,
-  }
-
-  def __init__(self, config_type: str) -> None:
-    """Sets concrete config type.
-
-    Args:
-        config_type: Type of config that should be built.
-
-    Raises:
-        GarfConfigException: When incorrect config_type is supplied.
-    """
-    if config_type not in self._config_mapping:
-      raise GarfConfigException(f'Invalid config_type: {config_type}')
-    self._config_type = config_type
-    self.config = self._config_mapping.get(config_type)
-
-  def build(
-    self, parameters: dict[str, str], cli_named_args: Sequence[str]
-  ) -> BaseConfig | None:
-    """Builds config from file, build from arguments or both ways.
-
-    When there are both config_file and CLI arguments the latter have more
-    priority.
-
-    Args:
-        parameters: Parsed CLI arguments.
-        cli_named_args: Unparsed CLI args in a form `--key.subkey=value`.
-
-    Returns:
-        Concrete config with injected values.
-    """
-    if not (garf_config_path := parameters.get('garf_config')):
-      return self._build_config(parameters, cli_named_args)
-    config_file = self._load_config(garf_config_path)
-    config_cli = self._build_config(
-      parameters, cli_named_args, init_defaults=False
-    )
-    if config_file and config_cli:
-      config_file = config_file + config_cli
-    return config_file
-
-  def _build_config(
-    self,
-    parameters: dict[str, str],
-    cli_named_args: Sequence[str],
-    init_defaults: bool = True,
-  ) -> BaseConfig | None:
-    """Builds config from named and unnamed CLI parameters.
-
-    Args:
-        parameters: Parsed CLI arguments.
-        cli_named_args: Unparsed CLI args in a form `--key.subkey=value`.
-        init_defaults: Whether to provided default config values if
-            expected parameter is missing
-
-    Returns:
-        Concrete config with injected values.
-    """
-    output = parameters.get('output')
-    config_parameters = {
-      k: v for k, v in parameters.items() if k in self.config.__annotations__
-    }
-    cli_params = ParamsParser(['macro', 'template', output]).parse(
-      cli_named_args
-    )
-    cli_params = _remove_empty_values(cli_params)
-    if output and (writer_params := cli_params.get(output)):
-      _ = cli_params.pop(output)
-      config_parameters.update({'writer_params': writer_params})
-    if cli_params:
-      config_parameters.update({'params': cli_params})
-    if not config_parameters:
-      return None
-    if init_defaults:
-      return self.config.from_dict(config_parameters)
-    return self.config(**config_parameters)
-
-  def _load_config(self, garf_config_path: str) -> BaseConfig:
-    """Loads config from provided path.
-
-    Args:
-        garf_config_path: Path to local or remote storage.
-
-    Returns:
-        Concreate config with values taken from config file.
-
-    Raises:
-        GarfConfigException:
-            If config file missing `garf` section.
-    """
-    with smart_open.open(garf_config_path, encoding='utf-8') as f:
-      config = yaml.safe_load(f)
-    garf_section = config.get(self._config_type)
-    if not garf_section:
-      raise GarfConfigException(
-        f'Invalid config, must have `{self._config_type}` section!'
-      )
-    config_parameters = {
-      k: v for k, v in garf_section.items() if k in self.config.__annotations__
-    }
-    if params := garf_section.get('params', {}):
-      config_parameters.update({'params': params})
-    if writer_params := garf_section.get(garf_section.get('output', '')):
-      config_parameters.update({'writer_params': writer_params})
-    return self.config(**config_parameters)
-
 
 class ParamsParser:
   def __init__(self, identifiers: Sequence[str]) -> None:
@@ -298,6 +51,8 @@ class ParamsParser:
     if not identifier or identifier not in key:
       return None
     provided_identifier, *keys = key.split('.')
+    if not keys:
+      return None
     if len(keys) > 1:
       raise GarfParamsException(
         f'{key} is invalid format,'
@@ -306,9 +61,10 @@ class ParamsParser:
       )
     provided_identifier = provided_identifier.replace('--', '')
     if provided_identifier not in self.identifiers:
+      supported_arguments = ', '.join(self.identifiers)
       raise GarfParamsException(
         f'CLI argument {provided_identifier} is not supported'
-        f', supported arguments {", ".join(self.identifiers)}'
+        f', supported arguments {supported_arguments}'
       )
     if provided_identifier != identifier:
       return None
@@ -334,118 +90,18 @@ class GarfParamsException(Exception):
   """Defines exception for incorrect parameters."""
 
 
-def convert_date(date_string: str) -> str:
-  """Converts specific dates parameters to actual dates.
-
-  Returns:
-      Date string in YYYY-MM-DD format.
-
-  Raises:
-      ValueError:
-          If dynamic lookback value (:YYYYMMDD-N) is incorrect.
-  """
-  if isinstance(date_string, list) or date_string.find(':YYYY') == -1:
-    return date_string
-  current_date = datetime.date.today()
-  date_object = date_string.split('-')
-  base_date = date_object[0]
-  if len(date_object) == 2:
-    try:
-      days_ago = int(date_object[1])
-    except ValueError as e:
-      raise ValueError(
-        'Must provide numeric value for a number lookback period, '
-        'i.e. :YYYYMMDD-1'
-      ) from e
-  else:
-    days_ago = 0
-  if base_date == ':YYYY':
-    new_date = datetime.datetime(current_date.year, 1, 1)
-    delta = relativedelta.relativedelta(years=days_ago)
-  elif base_date == ':YYYYMM':
-    new_date = datetime.datetime(current_date.year, current_date.month, 1)
-    delta = relativedelta.relativedelta(months=days_ago)
-  elif base_date == ':YYYYMMDD':
-    new_date = current_date
-    delta = relativedelta.relativedelta(days=days_ago)
-  return (new_date - delta).strftime('%Y-%m-%d')
-
-
-class ConfigSaver:
-  def __init__(self, path: str) -> None:
-    self.path = path
-
-  def save(self, garf_config: BaseConfig):
-    if os.path.exists(self.path):
-      with smart_open.open(self.path, 'r', encoding='utf-8') as f:
-        config = yaml.safe_load(f)
-    else:
-      config = {}
-    config = self.prepare_config(config, garf_config)
-    with smart_open.open(self.path, 'w', encoding='utf-8') as f:
-      yaml.dump(
-        config, f, default_flow_style=False, sort_keys=False, encoding='utf-8'
-      )
-
-  def prepare_config(self, config: dict, garf_config: BaseConfig) -> dict:
-    garf = dataclasses.asdict(garf_config)
-    if isinstance(garf_config, GarfConfig):
-      garf[garf_config.output] = garf_config.writer_params
-      if not isinstance(garf_config.account, MutableSequence):
-        garf['account'] = garf_config.account.split(',')
-      del garf['writer_params']
-      garf = _remove_empty_values(garf)
-      config.update({'garf': garf})
-    if isinstance(garf_config, GarfBqConfig):
-      garf = _remove_empty_values(garf)
-      config.update({'garf-bq': garf})
-    if isinstance(garf_config, GarfSqlConfig):
-      garf = _remove_empty_values(garf)
-      config.update({'garf-sql': garf})
-    return config
-
-
-def initialize_runtime_parameters(config: BaseConfig) -> BaseConfig:
-  """Formats parameters and add common parameter in config.
-
-  Initialization identifies whether there are `date` parameters and performs
-  necessary date conversions.
-  Set of parameters that need to be generally available are injected into
-  every parameter section of the config.
-
-  Args:
-      config: Instantiated config.
-
-  Returns:
-      Config with formatted parameters.
-  """
-  common_params = query_editor.CommonParametersMixin().common_params
-  for key, param in config.params.items():
-    for key_param, value_param in param.items():
-      config.params[key][key_param] = convert_date(value_param)
-    for common_param_key, common_param_value in common_params.items():
-      if common_param_key not in config.params[key]:
-        config.params[key][common_param_key] = common_param_value
-  return config
-
-
-def _remove_empty_values(dict_object: dict[str, Any]) -> dict[str, Any]:
-  """Remove all empty elements: strings, dictionaries from a dictionary."""
-  if isinstance(dict_object, dict):
-    return {
-      key: value
-      for key, value in (
-        (key, _remove_empty_values(value)) for key, value in dict_object.items()
-      )
-      if value
-    }
-  if isinstance(dict_object, (int, str, MutableSequence)):
-    return dict_object
+class LoggerEnum(str, enum.Enum):
+  local = 'local'
+  rich = 'rich'
+  gcloud = 'gcloud'
 
 
 def init_logging(
-  loglevel: str = 'INFO', logger_type: str = 'local', name: str = __name__
+  loglevel: str = 'INFO',
+  logger_type: str | LoggerEnum = 'local',
+  name: str = __name__,
 ) -> logging.Logger:
+  loglevel = getattr(logging, loglevel)
   if logger_type == 'rich':
     logging.basicConfig(
       format='%(message)s',
@@ -455,6 +111,23 @@ def init_logging(
         rich_logging.RichHandler(rich_tracebacks=True),
       ],
     )
+  elif logger_type == 'gcloud':
+    try:
+      import google.cloud.logging as glogging
+    except ImportError as e:
+      raise ImportError(
+        'Please install garf-executors with Cloud logging support - '
+        '`pip install garf-executors[bq]`'
+      ) from e
+
+    client = glogging.Client()
+    handler = glogging.handlers.CloudLoggingHandler(client, name=name)
+    handler.close()
+    glogging.handlers.setup_logging(handler, log_level=loglevel)
+    logging.basicConfig(
+      level=loglevel,
+      handlers=[handler],
+    )
   else:
     logging.basicConfig(
       format='[%(asctime)s][%(name)s][%(levelname)s] %(message)s',

garf_executors/execution_context.py

@@ -0,0 +1,83 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=C0330, g-bad-import-order, g-multiple-import
+
+"""Captures parameters for fetching data from APIs."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+
+import pydantic
+import smart_open
+import yaml
+from garf_core import query_editor
+from garf_io import writer
+from garf_io.writers import abs_writer
+
+
+class ExecutionContext(pydantic.BaseModel):
+  """Common context for executing one or more queries.
+
+  Attributes:
+    query_parameters: Parameters to dynamically change query text.
+    fetcher_parameters: Parameters to specify fetching setup.
+    writer: Type of writer to use.
+    writer_parameters: Optional parameters to setup writer.
+  """
+
+  query_parameters: query_editor.GarfQueryParameters | None = pydantic.Field(
+    default_factory=dict
+  )
+  fetcher_parameters: dict[str, str | bool | int | list[str | int]] | None = (
+    pydantic.Field(default_factory=dict)
+  )
+  writer: str | None = None
+  writer_parameters: dict[str, str] | None = pydantic.Field(
+    default_factory=dict
+  )
+
+  def model_post_init(self, __context__) -> None:
+    if self.fetcher_parameters is None:
+      self.fetcher_parameters = {}
+    if self.writer_parameters is None:
+      self.writer_parameters = {}
+    if not self.query_parameters:
+      self.query_parameters = query_editor.GarfQueryParameters()
+
+  @classmethod
+  def from_file(
+    cls, path: str | pathlib.Path | os.PathLike[str]
+  ) -> ExecutionContext:
+    """Builds context from local or remote yaml file."""
+    with smart_open.open(path, 'r', encoding='utf-8') as f:
+      data = yaml.safe_load(f)
+    return ExecutionContext(**data)
+
+  def save(self, path: str | pathlib.Path | os.PathLike[str]) -> str:
+    """Saves context to local or remote yaml file."""
+    with smart_open.open(path, 'w', encoding='utf-8') as f:
+      yaml.dump(self.model_dump(), f, encoding='utf-8')
+    return f'ExecutionContext is saved to {str(path)}'
+
+  @property
+  def writer_client(self) -> abs_writer.AbsWriter:
+    writer_client = writer.create_writer(self.writer, **self.writer_parameters)
+    if self.writer == 'bq':
+      _ = writer_client.create_or_get_dataset()
+    if self.writer == 'sheet':
+      writer_client.init_client()
+    return writer_client

garf_executors/executor.py

@@ -0,0 +1,87 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Defines common functionality between executors."""
+
+import asyncio
+
+from opentelemetry import trace
+
+from garf_executors import execution_context
+from garf_executors.telemetry import tracer
+
+
+class Executor:
+  """Defines common functionality between executors."""
+
+  @tracer.start_as_current_span('api.execute_batch')
+  def execute_batch(
+    self,
+    batch: dict[str, str],
+    context: execution_context.ExecutionContext,
+    parallel_threshold: int = 10,
+  ) -> list[str]:
+    """Executes batch of queries for a common context.
+
+    Args:
+      batch: Mapping between query_title and its text.
+      context: Execution context.
+      parallel_threshold: Number of queries to execute in parallel.
+
+    Returns:
+      Results of execution.
+    """
+    span = trace.get_current_span()
+    span.set_attribute('api.parallel_threshold', parallel_threshold)
+    return asyncio.run(
+      self._run(
+        batch=batch, context=context, parallel_threshold=parallel_threshold
+      )
+    )
+
+  async def aexecute(
+    self,
+    query: str,
+    title: str,
+    context: execution_context.ExecutionContext,
+  ) -> str:
+    """Performs query execution asynchronously.
+
+    Args:
+      query: Location of the query.
+      title: Name of the query.
+      context: Query execution context.
+
+    Returns:
+      Result of writing the report.
+    """
+    return await asyncio.to_thread(self.execute, query, title, context)
+
+  async def _run(
+    self,
+    batch: dict[str, str],
+    context: execution_context.ExecutionContext,
+    parallel_threshold: int,
+  ):
+    semaphore = asyncio.Semaphore(value=parallel_threshold)
+
+    async def run_with_semaphore(fn):
+      async with semaphore:
+        return await fn
+
+    tasks = [
+      self.aexecute(query=query, title=title, context=context)
+      for title, query in batch.items()
+    ]
+    return await asyncio.gather(*(run_with_semaphore(task) for task in tasks))