garf-executors 0.0.1__tar.gz

garf_executors-0.0.1/PKG-INFO

@@ -0,0 +1,30 @@
+Metadata-Version: 2.1
+Name: garf-executors
+Version: 0.0.1
+Summary: Executes queries against API and writes data to local/remote storage.
+Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>
+License: Apache 2.0
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: garf-core
+Requires-Dist: garf-io
+Provides-Extra: bq
+Requires-Dist: garf-io[bq]; extra == "bq"
+Requires-Dist: pandas; extra == "bq"
+Provides-Extra: sql
+Requires-Dist: garf-io[sqlalchemy]; extra == "sql"
+Requires-Dist: pandas; extra == "sql"
+Provides-Extra: all
+Requires-Dist: garf-executors[bq,sql]; extra == "all"
+
+# Gaarf Executors

garf_executors-0.0.1/README.md

@@ -0,0 +1 @@
+# Gaarf Executors

garf_executors-0.0.1/garf_executors/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Defines simplified import of executors.
+
+Instead of importing `garf_executors.api_executor.ApiQueryExecutor`
+import like this `garf_executors.ApiQueryExecutor`
+"""
+
+from __future__ import annotations
+
+from garf_executors.api_executor import ApiQueryExecutor
+
+__all__ = [
+  'ApiQueryExecutor',
+]

garf_executors-0.0.1/garf_executors/api_executor.py

@@ -0,0 +1,98 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for executing Garf queries and writing them to local/remote.
+
+ApiQueryExecutor performs fetching data from API in a form of
+GarfReport and saving it to local/remote storage.
+"""
+# pylint: disable=C0330, g-bad-import-order, g-multiple-import
+
+from __future__ import annotations
+
+import logging
+
+from garf_core import report_fetcher
+from garf_io.writers import abs_writer, console_writer
+
+logger = logging.getLogger(__name__)
+
+
+class ApiQueryExecutor:
+  """Gets data from API and writes them to local/remote storage.
+
+  Attributes:
+      api_client: a client used for connecting to API.
+  """
+
+  def __init__(self, fetcher: report_fetcher.ApiReportFetcher) -> None:
+    """Initializes QueryExecutor.
+
+    Args:
+        fetcher: Instantiated report fetcher.
+    """
+    self.fetcher = fetcher
+
+  async def aexecute(
+    self,
+    query_text: str,
+    query_name: str,
+    writer_client: abs_writer.AbsWriter = console_writer.ConsoleWriter(),
+    args: dict[str, str] | None = None,
+    **kwargs: str,
+  ) -> None:
+    """Reads query, extract results and stores them in a specified location.
+
+    Args:
+        query_text: Text for the query.
+        query_name: Identifier of a query.
+        customer_ids: All accounts for which query will be executed.
+        writer_client: Client responsible for writing data to local/remote
+            location.
+        args: Arguments that need to be passed to the query.
+        optimize_performance: strategy for speeding up query execution
+            ("NONE", "PROTOBUF", "BATCH", "BATCH_PROTOBUF").
+    """
+    self.execute(query_text, query_name, writer_client, args, **kwargs)
+
+  def execute(
+    self,
+    query_text: str,
+    query_name: str,
+    writer_client: abs_writer.AbsWriter = console_writer.ConsoleWriter(),
+    args: dict[str, str] | None = None,
+    **kwargs: str,
+  ) -> None:
+    """Reads query, extract results and stores them in a specified location.
+
+    Args:
+        query_text: Text for the query.
+        query_name: Identifier of a query.
+        writer_client: Client responsible for writing data to local/remote
+            location.
+        args: Arguments that need to be passed to the query.
+    """
+    results = self.fetcher.fetch(
+      query_specification=query_text, args=args, **kwargs
+    )
+    logger.debug(
+      'Start writing data for query %s via %s writer',
+      query_name,
+      type(writer_client),
+    )
+    writer_client.write(results, query_name)
+    logger.debug(
+      'Finish writing data for query %s via %s writer',
+      query_name,
+      type(writer_client),
+    )

garf_executors-0.0.1/garf_executors/bq_executor.py

@@ -0,0 +1,123 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for executing queries in BigQuery."""
+
+from __future__ import annotations
+
+try:
+  from google.cloud import bigquery  # type: ignore
+except ImportError as e:
+  raise ImportError(
+    'Please install garf-executors with BigQuery support '
+    '- `pip install garf-executors[bq]`'
+  ) from e
+
+import logging
+
+import pandas as pd
+from google.cloud import exceptions as google_cloud_exceptions
+
+from garf_core import query_editor
+
+logger = logging.getLogger(__name__)
+
+
+class BigQueryExecutorError(Exception):
+  """Error when executor fails to run query."""
+
+
+class BigQueryExecutor(query_editor.TemplateProcessorMixin):
+  """Handles query execution in BigQuery.
+
+  Attributes:
+      project_id: Google Cloud project id.
+      location: BigQuery dataset location.
+      client: BigQuery client.
+  """
+
+  def __init__(self, project_id: str, location: str | None = None) -> None:
+    """Initializes BigQueryExecutor.
+
+    Args:
+        project_id: Google Cloud project id.
+        location: BigQuery dataset location.
+    """
+    self.project_id = project_id
+    self.location = location
+
+  @property
+  def client(self) -> bigquery.Client:
+    """Instantiates bigquery client."""
+    return bigquery.Client(self.project_id)
+
+  def execute(
+    self, script_name: str, query_text: str, params: dict | None = None
+  ) -> pd.DataFrame:
+    """Executes query in BigQuery.
+
+    Args:
+        script_name: Script identifier.
+        query_text: Query to be executed.
+        params: Optional parameters to be replaced in query text.
+
+    Returns:
+        DataFrame if query returns some data otherwise empty DataFrame.
+    """
+    query_text = self.replace_params_template(query_text, params)
+    job = self.client.query(query_text)
+    try:
+      result = job.result()
+      logger.debug('%s launched successfully', script_name)
+      if result.total_rows:
+        return result.to_dataframe()
+      return pd.DataFrame()
+    except google_cloud_exceptions.GoogleCloudError as e:
+      raise BigQueryExecutorError(e) from e
+
+  def create_datasets(self, macros: dict | None) -> None:
+    """Creates datasets in BQ based on values in a dict.
+
+    If dict contains keys with 'dataset' in them, then values for such keys
+    are treated as dataset names.
+
+    Args:
+        macros: Mapping containing data for query execution.
+    """
+    if macros and (datasets := extract_datasets(macros)):
+      for dataset in datasets:
+        dataset_id = f'{self.project_id}.{dataset}'
+        try:
+          self.client.get_dataset(dataset_id)
+        except google_cloud_exceptions.NotFound:
+          bq_dataset = bigquery.Dataset(dataset_id)
+          bq_dataset.location = self.location
+          self.client.create_dataset(bq_dataset, timeout=30)
+          logger.debug('Created new dataset %s', dataset_id)
+
+
+def extract_datasets(macros: dict | None) -> list[str]:
+  """Finds dataset-related keys based on values in a dict.
+
+  If dict contains keys with 'dataset' in them, then values for such keys
+  are treated as dataset names.
+
+  Args:
+      macros: Mapping containing data for query execution.
+
+  Returns:
+      Possible names of datasets.
+  """
+  if not macros:
+    return []
+  return [value for macro, value in macros.items() if 'dataset' in macro]

garf_executors-0.0.1/garf_executors/entrypoints/cli/api.py

@@ -0,0 +1,213 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for defing `garf` CLI utility.
+
+`garf` allows to execute GAQL queries and store results in local/remote
+storage.
+"""
+
+from __future__ import annotations
+
+import argparse
+import functools
+import sys
+from collections.abc import MutableSequence
+from concurrent import futures
+from pathlib import Path
+
+import smart_open
+import yaml
+from garf import api_clients, exceptions, query_executor
+from garf.cli import utils
+from garf.io import reader, writer
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('query', nargs='*')
+  parser.add_argument('-c', '--config', dest='garf_config', default=None)
+  parser.add_argument('--account', dest='account', default=None)
+  parser.add_argument('--output', dest='output', default=None)
+  parser.add_argument('--input', dest='input', default='file')
+  parser.add_argument(
+    '--ads-config', dest='config', default=str(Path.home() / 'google-ads.yaml')
+  )
+  parser.add_argument('--api-version', dest='api_version', default=None)
+  parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
+  parser.add_argument('--logger', dest='logger', default='local')
+  parser.add_argument(
+    '--customer-ids-query', dest='customer_ids_query', default=None
+  )
+  parser.add_argument(
+    '--customer-ids-query-file', dest='customer_ids_query_file', default=None
+  )
+  parser.add_argument('--save-config', dest='save_config', action='store_true')
+  parser.add_argument(
+    '--no-save-config', dest='save_config', action='store_false'
+  )
+  parser.add_argument(
+    '--config-destination', dest='save_config_dest', default='config.yaml'
+  )
+  parser.add_argument(
+    '--parallel-queries', dest='parallel_queries', action='store_true'
+  )
+  parser.add_argument(
+    '--no-parallel-queries', dest='parallel_queries', action='store_false'
+  )
+  parser.add_argument(
+    '--optimize-performance', dest='optimize_performance', default='NONE'
+  )
+  parser.add_argument('--dry-run', dest='dry_run', action='store_true')
+  parser.add_argument(
+    '--disable-account-expansion',
+    dest='disable_account_expansion',
+    action='store_true',
+  )
+  parser.add_argument('-v', '--version', dest='version', action='store_true')
+  parser.add_argument(
+    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
+  )
+  parser.set_defaults(save_config=False)
+  parser.set_defaults(parallel_queries=True)
+  parser.set_defaults(dry_run=False)
+  parser.set_defaults(disable_account_expansion=False)
+  args = parser.parse_known_args()
+  main_args = args[0]
+
+  if main_args.version:
+    import pkg_resources
+
+    version = pkg_resources.require('google-ads-api-report-fetcher')[0].version
+    print(f'garf version {version}')
+    sys.exit()
+
+  logger = utils.init_logging(
+    loglevel=main_args.loglevel.upper(), logger_type=main_args.logger
+  )
+  if not main_args.query:
+    logger.error('Please provide one or more queries to run')
+    raise exceptions.GarfMissingQueryException(
+      'Please provide one or more queries to run'
+    )
+
+  with smart_open.open(main_args.config, 'r', encoding='utf-8') as f:
+    google_ads_config_dict = yaml.safe_load(f)
+
+  config = utils.ConfigBuilder('garf').build(vars(main_args), args[1])
+  if not config.account:
+    if mcc := google_ads_config_dict.get('login_customer_id'):
+      config.account = str(mcc)
+    else:
+      raise exceptions.GarfMissingAccountException(
+        'No account found, please specify via --account CLI flag'
+        'or add as login_customer_id in google-ads.yaml'
+      )
+  logger.debug('config: %s', config)
+
+  if main_args.save_config and not main_args.garf_config:
+    utils.ConfigSaver(main_args.save_config_dest).save(config)
+  if main_args.dry_run:
+    sys.exit()
+
+  if config.params:
+    config = utils.initialize_runtime_parameters(config)
+  logger.debug('initialized config: %s', config)
+
+  ads_client = api_clients.GoogleAdsApiClient(
+    config_dict=google_ads_config_dict,
+    version=config.api_version,
+    use_proto_plus=main_args.optimize_performance
+    not in ('PROTOBUF', 'BATCH_PROTOBUF'),
+  )
+  ads_query_executor = query_executor.AdsQueryExecutor(ads_client)
+  reader_factory = reader.ReaderFactory()
+  reader_client = reader_factory.create_reader(main_args.input)
+
+  if config.customer_ids_query:
+    customer_ids_query = config.customer_ids_query
+  elif config.customer_ids_query_file:
+    file_reader = reader_factory.create_reader('file')
+    customer_ids_query = file_reader.read(config.customer_ids_query_file)
+  else:
+    customer_ids_query = None
+
+  if main_args.disable_account_expansion:
+    logger.info(
+      'Skipping account expansion because of ' 'disable_account_expansion flag'
+    )
+    customer_ids = (
+      config.account
+      if isinstance(config.account, MutableSequence)
+      else [config.account]
+    )
+  else:
+    customer_ids = ads_query_executor.expand_mcc(
+      config.account, customer_ids_query
+    )
+  if not customer_ids:
+    logger.warning(
+      'Not a single under MCC %s is found that satisfies '
+      'the following customer_id query: "%s"',
+      config.account,
+      customer_ids_query,
+    )
+    sys.exit()
+  writer_client = writer.WriterFactory().create_writer(
+    config.output, **config.writer_params
+  )
+  if config.output == 'bq':
+    _ = writer_client.create_or_get_dataset()
+  if config.output == 'sheet':
+    writer_client.init_client()
+
+  logger.info(
+    'Total number of customer_ids is %d, accounts=[%s]',
+    len(customer_ids),
+    ','.join(map(str, customer_ids)),
+  )
+
+  if main_args.parallel_queries:
+    logger.info('Running queries in parallel')
+    with futures.ThreadPoolExecutor(main_args.parallel_threshold) as executor:
+      future_to_query = {
+        executor.submit(
+          ads_query_executor.execute,
+          reader_client.read(query),
+          query,
+          customer_ids,
+          writer_client,
+          config.params,
+          main_args.optimize_performance,
+        ): query
+        for query in main_args.query
+      }
+      for future in futures.as_completed(future_to_query):
+        query = future_to_query[future]
+        utils.garf_runner(query, future.result, logger)
+  else:
+    logger.info('Running queries sequentially')
+    for query in main_args.query:
+      callback = functools.partial(
+        ads_query_executor.execute,
+        reader_client.read(query),
+        query,
+        customer_ids,
+        writer_client,
+        config.params,
+        main_args.optimize_performance,
+      )
+      utils.garf_runner(query, callback, logger)
+
+
+if __name__ == '__main__':
+  main()

garf_executors-0.0.1/garf_executors/entrypoints/cli/bq.py

@@ -0,0 +1,112 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for defing `garf-bq` CLI utility.
+
+`garf-bq` allows to execute BigQuery queries based on Garf config.
+"""
+
+from __future__ import annotations
+
+import argparse
+import functools
+import sys
+from concurrent import futures
+
+from garf_writers import reader  # type: ignore
+
+from garf_executors import bq_executor
+from garf_executors.entrypoints import utils
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('query', nargs='+')
+  parser.add_argument('-c', '--config', dest='garf_config', default=None)
+  parser.add_argument('--project', dest='project')
+  parser.add_argument(
+    '--dataset-location', dest='dataset_location', default=None
+  )
+  parser.add_argument('--save-config', dest='save_config', action='store_true')
+  parser.add_argument(
+    '--no-save-config', dest='save_config', action='store_false'
+  )
+  parser.add_argument(
+    '--config-destination', dest='save_config_dest', default='config.yaml'
+  )
+  parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
+  parser.add_argument('--logger', dest='logger', default='local')
+  parser.add_argument('--dry-run', dest='dry_run', action='store_true')
+  parser.add_argument(
+    '--parallel-queries', dest='parallel_queries', action='store_true'
+  )
+  parser.add_argument(
+    '--no-parallel-queries', dest='parallel_queries', action='store_false'
+  )
+  parser.add_argument(
+    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
+  )
+  parser.set_defaults(save_config=False)
+  parser.set_defaults(dry_run=False)
+  parser.set_defaults(parallel_queries=True)
+  args = parser.parse_known_args()
+  main_args = args[0]
+
+  logger = utils.init_logging(
+    loglevel=main_args.loglevel.upper(), logger_type=main_args.logger
+  )
+  config = utils.ConfigBuilder('garf-bq').build(vars(main_args), args[1])
+  logger.debug('config: %s', config)
+  if main_args.save_config and not main_args.garf_config:
+    utils.ConfigSaver(main_args.save_config_dest).save(config)
+  if main_args.dry_run:
+    sys.exit()
+
+  config = utils.initialize_runtime_parameters(config)
+  logger.debug('initialized config: %s', config)
+
+  bigquery_executor = bq_executor.BigQueryExecutor(
+    project_id=config.project, location=config.dataset_location
+  )
+  bigquery_executor.create_datasets(config.params.get('macro'))
+
+  reader_client = reader.FileReader()
+
+  if main_args.parallel_queries:
+    logger.info('Running queries in parallel')
+    with futures.ThreadPoolExecutor(
+      max_workers=main_args.parallel_threshold
+    ) as executor:
+      future_to_query = {
+        executor.submit(
+          bigquery_executor.execute,
+          query,
+          reader_client.read(query),
+          config.params,
+        ): query
+        for query in sorted(main_args.query)
+      }
+      for future in futures.as_completed(future_to_query):
+        query = future_to_query[future]
+        utils.postprocessor_runner(query, future.result, logger)
+  else:
+    logger.info('Running queries sequentially')
+    for query in sorted(main_args.query):
+      callback = functools.partial(
+        executor.execute, query, reader_client.read(query), config.params
+      )
+      utils.postprocessor_runner(query, callback, logger)
+
+
+if __name__ == '__main__':
+  main()