garf-executors 0.0.2__py3-none-any.whl → 0.0.5__py3-none-any.whl

garf_executors/__init__.py

@@ -20,7 +20,11 @@ import like this `garf_executors.ApiQueryExecutor`
 from __future__ import annotations
 
 from garf_executors.api_executor import ApiQueryExecutor
+from garf_executors.fetchers import FETCHERS
 
 __all__ = [
+  'FETCHERS',
   'ApiQueryExecutor',
 ]
+
+__version__ = '0.0.5'

garf_executors/api_executor.py

@@ -22,12 +22,47 @@ from __future__ import annotations
 
 import logging
 
-from garf_core import report_fetcher
-from garf_io.writers import abs_writer, console_writer
+import pydantic
+
+from garf_core import query_editor, report_fetcher
+from garf_executors import exceptions
+from garf_io import writer
+from garf_io.writers import abs_writer
 
 logger = logging.getLogger(__name__)
 
 
+class ApiExecutionContext(pydantic.BaseModel):
+  """Common context for executing one or more queries.
+
+  Attributes:
+    query_parameters: Parameters to dynamically change query text.
+    fetcher_parameters: Parameters to specify fetching setup.
+    writer: Type of writer to use.
+    writer_parameters: Optional parameters to setup writer.
+  """
+
+  query_parameters: query_editor.GarfQueryParameters | None = None
+  fetcher_parameters: dict[str, str] | None = None
+  writer: str = 'console'
+  writer_parameters: dict[str, str] | None = None
+
+  def model_post_init(self, __context__) -> None:
+    if self.fetcher_parameters is None:
+      self.fetcher_parameters = {}
+    if self.writer_parameters is None:
+      self.writer_parameters = {}
+
+  @property
+  def writer_client(self) -> abs_writer.AbsWriter:
+    writer_client = writer.create_writer(self.writer, **self.writer_parameters)
+    if self.writer == 'bq':
+      _ = writer_client.create_or_get_dataset()
+    if self.writer == 'sheet':
+      writer_client.init_client()
+    return writer_client
+
+
 class ApiQueryExecutor:
   """Gets data from API and writes them to local/remote storage.
 
@@ -36,7 +71,7 @@ class ApiQueryExecutor:
   """
 
   def __init__(self, fetcher: report_fetcher.ApiReportFetcher) -> None:
-    """Initializes QueryExecutor.
+    """Initializes ApiQueryExecutor.
 
     Args:
         fetcher: Instantiated report fetcher.
@@ -44,55 +79,54 @@ class ApiQueryExecutor:
     self.fetcher = fetcher
 
   async def aexecute(
-    self,
-    query_text: str,
-    query_name: str,
-    writer_client: abs_writer.AbsWriter = console_writer.ConsoleWriter(),
-    args: dict[str, str] | None = None,
-    **kwargs: str,
+    self, query: str, context: ApiExecutionContext, **kwargs: str
   ) -> None:
     """Reads query, extract results and stores them in a specified location.
 
     Args:
-        query_text: Text for the query.
-        query_name: Identifier of a query.
-        customer_ids: All accounts for which query will be executed.
-        writer_client: Client responsible for writing data to local/remote
-            location.
-        args: Arguments that need to be passed to the query.
-        optimize_performance: strategy for speeding up query execution
-            ("NONE", "PROTOBUF", "BATCH", "BATCH_PROTOBUF").
+      query: Location of the query.
+      context: Query execution context.
     """
-    self.execute(query_text, query_name, writer_client, args, **kwargs)
+    self.execute(query, context, **kwargs)
 
   def execute(
     self,
-    query_text: str,
-    query_name: str,
-    writer_client: abs_writer.AbsWriter = console_writer.ConsoleWriter(),
-    args: dict[str, str] | None = None,
-    **kwargs: str,
+    query: str,
+    title: str,
+    context: ApiExecutionContext,
   ) -> None:
     """Reads query, extract results and stores them in a specified location.
 
     Args:
-        query_text: Text for the query.
-        query_name: Identifier of a query.
-        writer_client: Client responsible for writing data to local/remote
-            location.
-        args: Arguments that need to be passed to the query.
+      query: Location of the query.
+      title: Name of the query.
+      context: Query execution context.
+
+    Raises:
+      GarfExecutorError: When failed to execute query.
     """
-    results = self.fetcher.fetch(
-      query_specification=query_text, args=args, **kwargs
-    )
-    logger.debug(
-      'Start writing data for query %s via %s writer',
-      query_name,
-      type(writer_client),
-    )
-    writer_client.write(results, query_name)
-    logger.debug(
-      'Finish writing data for query %s via %s writer',
-      query_name,
-      type(writer_client),
-    )
+    try:
+      logger.debug('starting query %s', query)
+      results = self.fetcher.fetch(
+        query_specification=query,
+        args=context.query_parameters,
+        **context.fetcher_parameters,
+      )
+      writer_client = context.writer_client
+      logger.debug(
+        'Start writing data for query %s via %s writer',
+        title,
+        type(writer_client),
+      )
+      writer_client.write(results, title)
+      logger.debug(
+        'Finish writing data for query %s via %s writer',
+        title,
+        type(writer_client),
+      )
+      logger.info('%s executed successfully', title)
+    except Exception as e:
+      logger.error('%s generated an exception: %s', title, str(e))
+      raise exceptions.GarfExecutorError(
+        '%s generated an exception: %s', title, str(e)
+      ) from e

garf_executors/entrypoints/cli.py

@@ -0,0 +1,115 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for defining `garf` CLI utility.
+
+`garf` allows to execute queries and store results in local/remote
+storage.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from concurrent import futures
+
+import garf_executors
+from garf_executors import exceptions
+from garf_executors.entrypoints import utils
+from garf_io import reader
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('query', nargs='*')
+  parser.add_argument('-c', '--config', dest='garf_config', default=None)
+  parser.add_argument('--source', dest='source', default=None)
+  parser.add_argument('--output', dest='output', default='console')
+  parser.add_argument('--input', dest='input', default='file')
+  parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
+  parser.add_argument('--logger', dest='logger', default='local')
+  parser.add_argument(
+    '--parallel-queries', dest='parallel_queries', action='store_true'
+  )
+  parser.add_argument(
+    '--no-parallel-queries', dest='parallel_queries', action='store_false'
+  )
+  parser.add_argument('--dry-run', dest='dry_run', action='store_true')
+  parser.add_argument('-v', '--version', dest='version', action='store_true')
+  parser.add_argument(
+    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
+  )
+  parser.set_defaults(parallel_queries=True)
+  parser.set_defaults(dry_run=False)
+  args, kwargs = parser.parse_known_args()
+
+  if args.version:
+    print(garf_executors.__version__)
+    sys.exit()
+  if not (source := args.source):
+    raise exceptions.GarfExecutorError(
+      f'Select one of available sources: {list(garf_executors.FETCHERS.keys())}'
+    )
+  if not (concrete_api_fetcher := garf_executors.FETCHERS.get(source)):
+    raise exceptions.GarfExecutorError(f'Source {source} is not available.')
+
+  logger = utils.init_logging(
+    loglevel=args.loglevel.upper(), logger_type=args.logger
+  )
+  if not args.query:
+    logger.error('Please provide one or more queries to run')
+    raise exceptions.GarfExecutorError(
+      'Please provide one or more queries to run'
+    )
+  config = utils.ConfigBuilder('garf').build(vars(args), kwargs)
+  logger.debug('config: %s', config)
+
+  if config.params:
+    config = utils.initialize_runtime_parameters(config)
+  logger.debug('initialized config: %s', config)
+
+  extra_parameters = utils.ParamsParser(['source']).parse(kwargs)
+  source_parameters = extra_parameters.get('source', {})
+  reader_client = reader.create_reader(args.input)
+
+  context = garf_executors.api_executor.ApiExecutionContext(
+    query_parameters=config.params,
+    writer=args.output,
+    writer_parameters=config.writer_params,
+    fetcher_parameters=source_parameters,
+  )
+  query_executor = garf_executors.api_executor.ApiQueryExecutor(
+    concrete_api_fetcher(**source_parameters)
+  )
+  if args.parallel_queries:
+    logger.info('Running queries in parallel')
+    with futures.ThreadPoolExecutor(args.parallel_threshold) as executor:
+      future_to_query = {
+        executor.submit(
+          query_executor.execute,
+          reader_client.read(query),
+          query,
+          context,
+        ): query
+        for query in args.query
+      }
+      for future in futures.as_completed(future_to_query):
+        future.result()
+  else:
+    logger.info('Running queries sequentially')
+    for query in args.query:
+      query_executor.execute(reader_client.read(query), query, context)
+
+
+if __name__ == '__main__':
+  main()

garf_executors/entrypoints/server.py

@@ -0,0 +1,65 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""FastAPI endpoint for executing queries."""
+
+import fastapi
+import pydantic
+import uvicorn
+
+import garf_executors
+from garf_executors import exceptions
+
+
+class ApiExecutorRequest(pydantic.BaseModel):
+  """Request for executing a query.
+
+  Attributes:
+    source: Type of API to interact with.
+    query: Query to execute.
+    title: Name of the query used as an output for writing.
+    context: Execution context.
+  """
+
+  source: str
+  query: str
+  title: str
+  context: garf_executors.api_executor.ApiExecutionContext
+
+
+router = fastapi.APIRouter(prefix='/api')
+
+
+@router.post('/execute')
+async def execute(request: ApiExecutorRequest) -> dict[str, str]:
+  if not (concrete_api_fetcher := garf_executors.FETCHERS.get(request.source)):
+    raise exceptions.GarfExecutorError(
+      f'Source {request.source} is not available.'
+    )
+
+  query_executor = garf_executors.api_executor.ApiQueryExecutor(
+    concrete_api_fetcher(**request.context.fetcher_parameters)
+  )
+
+  query_executor.execute(request.query, request.title, request.context)
+
+  return fastapi.responses.JSONResponse(
+    content=fastapi.encoders.jsonable_encoder({'result': 'success'})
+  )
+
+
+if __name__ == '__main__':
+  app = fastapi.FastAPI()
+  app.include_router(router)
+  uvicorn.run(app)

garf_executors/entrypoints/utils.py

@@ -20,9 +20,8 @@ import datetime
 import logging
 import os
 import sys
-import traceback
 from collections.abc import MutableSequence, Sequence
-from typing import Any, Callable, TypedDict
+from typing import Any, TypedDict
 
 import smart_open
 import yaml
@@ -298,21 +297,36 @@ class ParamsParser:
     key = param[0]
     if not identifier or identifier not in key:
       return None
-    provided_identifier, key = key.split('.')
+    provided_identifier, *keys = key.split('.')
+    if len(keys) > 1:
+      raise GarfParamsException(
+        f'{key} is invalid format,'
+        f'`--{identifier}.key=value` or `--{identifier}.key` '
+        'are the correct formats'
+      )
     provided_identifier = provided_identifier.replace('--', '')
     if provided_identifier not in self.identifiers:
       raise GarfParamsException(
         f'CLI argument {provided_identifier} is not supported'
-        f", supported arguments {', '.join(self.identifiers)}"
+        f', supported arguments {", ".join(self.identifiers)}'
       )
     if provided_identifier != identifier:
       return None
-    key = key.replace('-', '_')
+    key = keys[0].replace('-', '_')
+    if not key:
+      raise GarfParamsException(
+        f'{identifier} {key} is invalid,'
+        f'`--{identifier}.key=value` or `--{identifier}.key` '
+        'are the correct formats'
+      )
     if len(param) == 2:
       return {key: param[1]}
+    if len(param) == 1:
+      return {key: True}
     raise GarfParamsException(
       f'{identifier} {key} is invalid,'
-      f'--{identifier}.key=value is the correct format'
+      f'`--{identifier}.key=value` or `--{identifier}.key` '
+      'are the correct formats'
     )
 
 
@@ -429,25 +443,6 @@ def _remove_empty_values(dict_object: dict[str, Any]) -> dict[str, Any]:
     return dict_object
 
 
-def garf_runner(query: str, callback: Callable, logger) -> None:
-  try:
-    logger.debug('starting query %s', query)
-    callback()
-    logger.info('%s executed successfully', query)
-  except Exception as e:
-    traceback.print_tb(e.__traceback__)
-    logger.error('%s generated an exception: %s', query, str(e))
-
-
-def postprocessor_runner(query: str, callback: Callable, logger) -> None:
-  try:
-    logger.debug('starting query %s', query)
-    callback()
-    logger.info('%s executed successfully', query)
-  except Exception as e:
-    logger.error('%s generated an exception: %s', query, str(e))
-
-
 def init_logging(
   loglevel: str = 'INFO', logger_type: str = 'local', name: str = __name__
 ) -> logging.Logger:

garf_executors/exceptions.py

@@ -0,0 +1,17 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class GarfExecutorError(Exception):
+  """Base class for garf executor exceptions."""

garf_executors/fetchers.py

@@ -0,0 +1,37 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+from importlib.metadata import entry_points
+
+from garf_core import report_fetcher
+
+
+def get_report_fetchers() -> dict[str, report_fetcher.ApiReportFetcher]:
+  fetchers = entry_points(group='garf')
+  found_fetchers = {}
+  for fetcher in fetchers:
+    try:
+      fetcher_module = fetcher.load()
+      for name, obj in inspect.getmembers(fetcher_module):
+        if inspect.isclass(obj) and issubclass(
+          obj, report_fetcher.ApiReportFetcher
+        ):
+          found_fetchers[fetcher.name] = getattr(fetcher_module, name)
+    except ModuleNotFoundError:
+      continue
+  return found_fetchers
+
+
+FETCHERS = get_report_fetchers()

garf_executors-0.0.5.dist-info/METADATA

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: garf-executors
+Version: 0.0.5
+Summary: Executes queries against API and writes data to local/remote storage.
+Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
+License: Apache 2.0
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: Apache Software License
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: garf-core
+Requires-Dist: garf-io
+Requires-Dist: pyyaml
+Requires-Dist: pydantic
+Provides-Extra: bq
+Requires-Dist: garf-io[bq]; extra == "bq"
+Requires-Dist: pandas; extra == "bq"
+Provides-Extra: sql
+Requires-Dist: garf-io[sqlalchemy]; extra == "sql"
+Requires-Dist: pandas; extra == "sql"
+Provides-Extra: server
+Requires-Dist: fastapi[standard]; extra == "server"
+Provides-Extra: all
+Requires-Dist: garf-executors[bq,server,sql]; extra == "all"
+
+# `garf-executors` - One stop-shop for interacting with Reporting APIs.
+
+`garf-executors` is responsible for orchestrating process of fetching from API and storing data in a storage.
+
+Currently the following executors are supports:
+
+* `ApiExecutor` - fetching data from reporting API and saves it to a requested destination.
+* `BigQueryExecutor` - executes SQL code in BigQuery.
+* `SqlExecutor` - executes SQL code in a SqlAlchemy supported DB.
+
+## Installation
+
+`pip install garf-executors`
+
+## Usage
+
+After `garf-executors` is installed you can use `garf` utility to perform fetching.
+
+```
+garf <QUERIES> --source <API_SOURCE> \
+  --output <OUTPUT_TYPE> \
+  --source.params1=<VALUE>
+```
+
+where
+
+* `<QUERIES>`- local or remote path(s) to files with queries.
+* `<API_SOURCE>`- type of API to use. Based on that the appropriate report fetcher will be initialized.
+* `<OUTPUT_TYPE>` - output supported by [`garf-io` library](../garf_io/README.md).
+
+If your report fetcher requires additional parameters you can pass them via key value pairs under `--source.` argument, i.e.`--source.regionCode='US'` - to get data only from *US*.
+> Concrete `--source` parameters are dependent on a particular report fetcher and should be looked up in a documentation for this fetcher.

garf_executors-0.0.5.dist-info/RECORD

@@ -0,0 +1,15 @@
+garf_executors/__init__.py,sha256=Afg7pEjgXQlhiQ2hCTlef77ypbLRt6i_8gB1tIv7rUM,955
+garf_executors/api_executor.py,sha256=udrlMiYUmKh5NsIuJkNowqCenvtf5O925FPFawXSXbM,4021
+garf_executors/bq_executor.py,sha256=JBPxbDRYgUgpJv6SqYiFPypTFjZGIZ-SOOb6dS2sZQY,3822
+garf_executors/exceptions.py,sha256=U_7Q2ZMOUf89gzZd2pw7y3g7i1NeByPPKfpZ3q7p3ZU,662
+garf_executors/fetchers.py,sha256=gkAKHsDPzJySg4wYLZeCmNINtk6f17-jFzOP7tE82r8,1226
+garf_executors/sql_executor.py,sha256=vBNQ4HZZYxP_EYAh8Z4BerzLESfsNpXdhENzXIw-OIo,2532
+garf_executors/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+garf_executors/entrypoints/cli.py,sha256=mWvPQkaqarDj5byHRvNAweVbUQiHZLXrC-35zY7l4fs,4043
+garf_executors/entrypoints/server.py,sha256=rJ29VKWKaYJci1BLxZx-0LSILmUMf5BK8G1RRjRS2ts,1836
+garf_executors/entrypoints/utils.py,sha256=ZZJFe2N4KwgzPRvak9gW_B25qESnzOyuF-qYZ2wW2_M,14974
+garf_executors-0.0.5.dist-info/METADATA,sha256=a_7xXfg6sI-ugJAEvGuuXKym1nYSp4WvjuR4AypboEM,2648
+garf_executors-0.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+garf_executors-0.0.5.dist-info/entry_points.txt,sha256=LskWNFIw8j0WJuI18-32OZrlASXAMg1XtrRYwsKBz2E,61
+garf_executors-0.0.5.dist-info/top_level.txt,sha256=sP4dCXOENPn1hDFAunjMV8Js4NND_KGeO_gQWuaT0EY,15
+garf_executors-0.0.5.dist-info/RECORD,,

{garf_executors-0.0.2.dist-info → garf_executors-0.0.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

garf_executors-0.0.5.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+garf = garf_executors.entrypoints.cli:main

garf_executors/entrypoints/cli/api.py

@@ -1,213 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Module for defing `garf` CLI utility.
-
-`garf` allows to execute GAQL queries and store results in local/remote
-storage.
-"""
-
-from __future__ import annotations
-
-import argparse
-import functools
-import sys
-from collections.abc import MutableSequence
-from concurrent import futures
-from pathlib import Path
-
-import smart_open
-import yaml
-from garf import api_clients, exceptions, query_executor
-from garf.cli import utils
-from garf.io import reader, writer
-
-
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('query', nargs='*')
-  parser.add_argument('-c', '--config', dest='garf_config', default=None)
-  parser.add_argument('--account', dest='account', default=None)
-  parser.add_argument('--output', dest='output', default=None)
-  parser.add_argument('--input', dest='input', default='file')
-  parser.add_argument(
-    '--ads-config', dest='config', default=str(Path.home() / 'google-ads.yaml')
-  )
-  parser.add_argument('--api-version', dest='api_version', default=None)
-  parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
-  parser.add_argument('--logger', dest='logger', default='local')
-  parser.add_argument(
-    '--customer-ids-query', dest='customer_ids_query', default=None
-  )
-  parser.add_argument(
-    '--customer-ids-query-file', dest='customer_ids_query_file', default=None
-  )
-  parser.add_argument('--save-config', dest='save_config', action='store_true')
-  parser.add_argument(
-    '--no-save-config', dest='save_config', action='store_false'
-  )
-  parser.add_argument(
-    '--config-destination', dest='save_config_dest', default='config.yaml'
-  )
-  parser.add_argument(
-    '--parallel-queries', dest='parallel_queries', action='store_true'
-  )
-  parser.add_argument(
-    '--no-parallel-queries', dest='parallel_queries', action='store_false'
-  )
-  parser.add_argument(
-    '--optimize-performance', dest='optimize_performance', default='NONE'
-  )
-  parser.add_argument('--dry-run', dest='dry_run', action='store_true')
-  parser.add_argument(
-    '--disable-account-expansion',
-    dest='disable_account_expansion',
-    action='store_true',
-  )
-  parser.add_argument('-v', '--version', dest='version', action='store_true')
-  parser.add_argument(
-    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
-  )
-  parser.set_defaults(save_config=False)
-  parser.set_defaults(parallel_queries=True)
-  parser.set_defaults(dry_run=False)
-  parser.set_defaults(disable_account_expansion=False)
-  args = parser.parse_known_args()
-  main_args = args[0]
-
-  if main_args.version:
-    import pkg_resources
-
-    version = pkg_resources.require('google-ads-api-report-fetcher')[0].version
-    print(f'garf version {version}')
-    sys.exit()
-
-  logger = utils.init_logging(
-    loglevel=main_args.loglevel.upper(), logger_type=main_args.logger
-  )
-  if not main_args.query:
-    logger.error('Please provide one or more queries to run')
-    raise exceptions.GarfMissingQueryException(
-      'Please provide one or more queries to run'
-    )
-
-  with smart_open.open(main_args.config, 'r', encoding='utf-8') as f:
-    google_ads_config_dict = yaml.safe_load(f)
-
-  config = utils.ConfigBuilder('garf').build(vars(main_args), args[1])
-  if not config.account:
-    if mcc := google_ads_config_dict.get('login_customer_id'):
-      config.account = str(mcc)
-    else:
-      raise exceptions.GarfMissingAccountException(
-        'No account found, please specify via --account CLI flag'
-        'or add as login_customer_id in google-ads.yaml'
-      )
-  logger.debug('config: %s', config)
-
-  if main_args.save_config and not main_args.garf_config:
-    utils.ConfigSaver(main_args.save_config_dest).save(config)
-  if main_args.dry_run:
-    sys.exit()
-
-  if config.params:
-    config = utils.initialize_runtime_parameters(config)
-  logger.debug('initialized config: %s', config)
-
-  ads_client = api_clients.GoogleAdsApiClient(
-    config_dict=google_ads_config_dict,
-    version=config.api_version,
-    use_proto_plus=main_args.optimize_performance
-    not in ('PROTOBUF', 'BATCH_PROTOBUF'),
-  )
-  ads_query_executor = query_executor.AdsQueryExecutor(ads_client)
-  reader_factory = reader.ReaderFactory()
-  reader_client = reader_factory.create_reader(main_args.input)
-
-  if config.customer_ids_query:
-    customer_ids_query = config.customer_ids_query
-  elif config.customer_ids_query_file:
-    file_reader = reader_factory.create_reader('file')
-    customer_ids_query = file_reader.read(config.customer_ids_query_file)
-  else:
-    customer_ids_query = None
-
-  if main_args.disable_account_expansion:
-    logger.info(
-      'Skipping account expansion because of ' 'disable_account_expansion flag'
-    )
-    customer_ids = (
-      config.account
-      if isinstance(config.account, MutableSequence)
-      else [config.account]
-    )
-  else:
-    customer_ids = ads_query_executor.expand_mcc(
-      config.account, customer_ids_query
-    )
-  if not customer_ids:
-    logger.warning(
-      'Not a single under MCC %s is found that satisfies '
-      'the following customer_id query: "%s"',
-      config.account,
-      customer_ids_query,
-    )
-    sys.exit()
-  writer_client = writer.WriterFactory().create_writer(
-    config.output, **config.writer_params
-  )
-  if config.output == 'bq':
-    _ = writer_client.create_or_get_dataset()
-  if config.output == 'sheet':
-    writer_client.init_client()
-
-  logger.info(
-    'Total number of customer_ids is %d, accounts=[%s]',
-    len(customer_ids),
-    ','.join(map(str, customer_ids)),
-  )
-
-  if main_args.parallel_queries:
-    logger.info('Running queries in parallel')
-    with futures.ThreadPoolExecutor(main_args.parallel_threshold) as executor:
-      future_to_query = {
-        executor.submit(
-          ads_query_executor.execute,
-          reader_client.read(query),
-          query,
-          customer_ids,
-          writer_client,
-          config.params,
-          main_args.optimize_performance,
-        ): query
-        for query in main_args.query
-      }
-      for future in futures.as_completed(future_to_query):
-        query = future_to_query[future]
-        utils.garf_runner(query, future.result, logger)
-  else:
-    logger.info('Running queries sequentially')
-    for query in main_args.query:
-      callback = functools.partial(
-        ads_query_executor.execute,
-        reader_client.read(query),
-        query,
-        customer_ids,
-        writer_client,
-        config.params,
-        main_args.optimize_performance,
-      )
-      utils.garf_runner(query, callback, logger)
-
-
-if __name__ == '__main__':
-  main()

garf_executors/entrypoints/cli/bq.py

@@ -1,112 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Module for defing `garf-bq` CLI utility.
-
-`garf-bq` allows to execute BigQuery queries based on Garf config.
-"""
-
-from __future__ import annotations
-
-import argparse
-import functools
-import sys
-from concurrent import futures
-
-from garf_writers import reader  # type: ignore
-
-from garf_executors import bq_executor
-from garf_executors.entrypoints import utils
-
-
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('query', nargs='+')
-  parser.add_argument('-c', '--config', dest='garf_config', default=None)
-  parser.add_argument('--project', dest='project')
-  parser.add_argument(
-    '--dataset-location', dest='dataset_location', default=None
-  )
-  parser.add_argument('--save-config', dest='save_config', action='store_true')
-  parser.add_argument(
-    '--no-save-config', dest='save_config', action='store_false'
-  )
-  parser.add_argument(
-    '--config-destination', dest='save_config_dest', default='config.yaml'
-  )
-  parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
-  parser.add_argument('--logger', dest='logger', default='local')
-  parser.add_argument('--dry-run', dest='dry_run', action='store_true')
-  parser.add_argument(
-    '--parallel-queries', dest='parallel_queries', action='store_true'
-  )
-  parser.add_argument(
-    '--no-parallel-queries', dest='parallel_queries', action='store_false'
-  )
-  parser.add_argument(
-    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
-  )
-  parser.set_defaults(save_config=False)
-  parser.set_defaults(dry_run=False)
-  parser.set_defaults(parallel_queries=True)
-  args = parser.parse_known_args()
-  main_args = args[0]
-
-  logger = utils.init_logging(
-    loglevel=main_args.loglevel.upper(), logger_type=main_args.logger
-  )
-  config = utils.ConfigBuilder('garf-bq').build(vars(main_args), args[1])
-  logger.debug('config: %s', config)
-  if main_args.save_config and not main_args.garf_config:
-    utils.ConfigSaver(main_args.save_config_dest).save(config)
-  if main_args.dry_run:
-    sys.exit()
-
-  config = utils.initialize_runtime_parameters(config)
-  logger.debug('initialized config: %s', config)
-
-  bigquery_executor = bq_executor.BigQueryExecutor(
-    project_id=config.project, location=config.dataset_location
-  )
-  bigquery_executor.create_datasets(config.params.get('macro'))
-
-  reader_client = reader.FileReader()
-
-  if main_args.parallel_queries:
-    logger.info('Running queries in parallel')
-    with futures.ThreadPoolExecutor(
-      max_workers=main_args.parallel_threshold
-    ) as executor:
-      future_to_query = {
-        executor.submit(
-          bigquery_executor.execute,
-          query,
-          reader_client.read(query),
-          config.params,
-        ): query
-        for query in sorted(main_args.query)
-      }
-      for future in futures.as_completed(future_to_query):
-        query = future_to_query[future]
-        utils.postprocessor_runner(query, future.result, logger)
-  else:
-    logger.info('Running queries sequentially')
-    for query in sorted(main_args.query):
-      callback = functools.partial(
-        executor.execute, query, reader_client.read(query), config.params
-      )
-      utils.postprocessor_runner(query, callback, logger)
-
-
-if __name__ == '__main__':
-  main()

garf_executors/entrypoints/cli/gaarf.py

@@ -1,213 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Module for defing `garf` CLI utility.
-
-`garf` allows to execute GAQL queries and store results in local/remote
-storage.
-"""
-
-from __future__ import annotations
-
-import argparse
-import functools
-import sys
-from collections.abc import MutableSequence
-from concurrent import futures
-from pathlib import Path
-
-import smart_open
-import yaml
-from garf import api_clients, exceptions, query_executor
-from garf.cli import utils
-from garf.io import reader, writer
-
-
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('query', nargs='*')
-  parser.add_argument('-c', '--config', dest='garf_config', default=None)
-  parser.add_argument('--account', dest='account', default=None)
-  parser.add_argument('--output', dest='output', default=None)
-  parser.add_argument('--input', dest='input', default='file')
-  parser.add_argument(
-    '--ads-config', dest='config', default=str(Path.home() / 'google-ads.yaml')
-  )
-  parser.add_argument('--api-version', dest='api_version', default=None)
-  parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
-  parser.add_argument('--logger', dest='logger', default='local')
-  parser.add_argument(
-    '--customer-ids-query', dest='customer_ids_query', default=None
-  )
-  parser.add_argument(
-    '--customer-ids-query-file', dest='customer_ids_query_file', default=None
-  )
-  parser.add_argument('--save-config', dest='save_config', action='store_true')
-  parser.add_argument(
-    '--no-save-config', dest='save_config', action='store_false'
-  )
-  parser.add_argument(
-    '--config-destination', dest='save_config_dest', default='config.yaml'
-  )
-  parser.add_argument(
-    '--parallel-queries', dest='parallel_queries', action='store_true'
-  )
-  parser.add_argument(
-    '--no-parallel-queries', dest='parallel_queries', action='store_false'
-  )
-  parser.add_argument(
-    '--optimize-performance', dest='optimize_performance', default='NONE'
-  )
-  parser.add_argument('--dry-run', dest='dry_run', action='store_true')
-  parser.add_argument(
-    '--disable-account-expansion',
-    dest='disable_account_expansion',
-    action='store_true',
-  )
-  parser.add_argument('-v', '--version', dest='version', action='store_true')
-  parser.add_argument(
-    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
-  )
-  parser.set_defaults(save_config=False)
-  parser.set_defaults(parallel_queries=True)
-  parser.set_defaults(dry_run=False)
-  parser.set_defaults(disable_account_expansion=False)
-  args = parser.parse_known_args()
-  main_args = args[0]
-
-  if main_args.version:
-    import pkg_resources
-
-    version = pkg_resources.require('google-ads-api-report-fetcher')[0].version
-    print(f'garf version {version}')
-    sys.exit()
-
-  logger = utils.init_logging(
-    loglevel=main_args.loglevel.upper(), logger_type=main_args.logger
-  )
-  if not main_args.query:
-    logger.error('Please provide one or more queries to run')
-    raise exceptions.GarfMissingQueryException(
-      'Please provide one or more queries to run'
-    )
-
-  with smart_open.open(main_args.config, 'r', encoding='utf-8') as f:
-    google_ads_config_dict = yaml.safe_load(f)
-
-  config = utils.ConfigBuilder('garf').build(vars(main_args), args[1])
-  if not config.account:
-    if mcc := google_ads_config_dict.get('login_customer_id'):
-      config.account = str(mcc)
-    else:
-      raise exceptions.GarfMissingAccountException(
-        'No account found, please specify via --account CLI flag'
-        'or add as login_customer_id in google-ads.yaml'
-      )
-  logger.debug('config: %s', config)
-
-  if main_args.save_config and not main_args.garf_config:
-    utils.ConfigSaver(main_args.save_config_dest).save(config)
-  if main_args.dry_run:
-    sys.exit()
-
-  if config.params:
-    config = utils.initialize_runtime_parameters(config)
-  logger.debug('initialized config: %s', config)
-
-  ads_client = api_clients.GoogleAdsApiClient(
-    config_dict=google_ads_config_dict,
-    version=config.api_version,
-    use_proto_plus=main_args.optimize_performance
-    not in ('PROTOBUF', 'BATCH_PROTOBUF'),
-  )
-  ads_query_executor = query_executor.AdsQueryExecutor(ads_client)
-  reader_factory = reader.ReaderFactory()
-  reader_client = reader_factory.create_reader(main_args.input)
-
-  if config.customer_ids_query:
-    customer_ids_query = config.customer_ids_query
-  elif config.customer_ids_query_file:
-    file_reader = reader_factory.create_reader('file')
-    customer_ids_query = file_reader.read(config.customer_ids_query_file)
-  else:
-    customer_ids_query = None
-
-  if main_args.disable_account_expansion:
-    logger.info(
-      'Skipping account expansion because of ' 'disable_account_expansion flag'
-    )
-    customer_ids = (
-      config.account
-      if isinstance(config.account, MutableSequence)
-      else [config.account]
-    )
-  else:
-    customer_ids = ads_query_executor.expand_mcc(
-      config.account, customer_ids_query
-    )
-  if not customer_ids:
-    logger.warning(
-      'Not a single under MCC %s is found that satisfies '
-      'the following customer_id query: "%s"',
-      config.account,
-      customer_ids_query,
-    )
-    sys.exit()
-  writer_client = writer.WriterFactory().create_writer(
-    config.output, **config.writer_params
-  )
-  if config.output == 'bq':
-    _ = writer_client.create_or_get_dataset()
-  if config.output == 'sheet':
-    writer_client.init_client()
-
-  logger.info(
-    'Total number of customer_ids is %d, accounts=[%s]',
-    len(customer_ids),
-    ','.join(map(str, customer_ids)),
-  )
-
-  if main_args.parallel_queries:
-    logger.info('Running queries in parallel')
-    with futures.ThreadPoolExecutor(main_args.parallel_threshold) as executor:
-      future_to_query = {
-        executor.submit(
-          ads_query_executor.execute,
-          reader_client.read(query),
-          query,
-          customer_ids,
-          writer_client,
-          config.params,
-          main_args.optimize_performance,
-        ): query
-        for query in main_args.query
-      }
-      for future in futures.as_completed(future_to_query):
-        query = future_to_query[future]
-        utils.garf_runner(query, future.result, logger)
-  else:
-    logger.info('Running queries sequentially')
-    for query in main_args.query:
-      callback = functools.partial(
-        ads_query_executor.execute,
-        reader_client.read(query),
-        query,
-        customer_ids,
-        writer_client,
-        config.params,
-        main_args.optimize_performance,
-      )
-      utils.garf_runner(query, callback, logger)
-
-
-if __name__ == '__main__':
-  main()

garf_executors/entrypoints/cli/sql.py

@@ -1,109 +0,0 @@
-# Copyright 2023 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Module for defing `garf` CLI utility.
-
-`garf-sql` allows to execute SQL queries in various Databases via SqlAlchemy.
-"""
-
-from __future__ import annotations
-
-import argparse
-import functools
-import sys
-from concurrent import futures
-
-import sqlalchemy
-from garf_writers import reader  # type: ignore
-
-from garf_executors import sql_executor
-from garf_executors.entrypoints import utils
-
-
-def main():
-  parser = argparse.ArgumentParser()
-  parser.add_argument('query', nargs='+')
-  parser.add_argument('-c', '--config', dest='garf_config', default=None)
-  parser.add_argument('--conn', '--connection-string', dest='connection_string')
-  parser.add_argument('--save-config', dest='save_config', action='store_true')
-  parser.add_argument(
-    '--no-save-config', dest='save_config', action='store_false'
-  )
-  parser.add_argument(
-    '--config-destination', dest='save_config_dest', default='config.yaml'
-  )
-  parser.add_argument('--log', '--loglevel', dest='loglevel', default='info')
-  parser.add_argument('--logger', dest='logger', default='local')
-  parser.add_argument('--dry-run', dest='dry_run', action='store_true')
-  parser.add_argument(
-    '--parallel-queries', dest='parallel_queries', action='store_true'
-  )
-  parser.add_argument(
-    '--no-parallel-queries', dest='parallel_queries', action='store_false'
-  )
-  parser.add_argument(
-    '--parallel-threshold', dest='parallel_threshold', default=None, type=int
-  )
-  parser.set_defaults(save_config=False)
-  parser.set_defaults(dry_run=False)
-  parser.set_defaults(parallel_queries=True)
-  args = parser.parse_known_args()
-  main_args = args[0]
-
-  logger = utils.init_logging(
-    loglevel=main_args.loglevel.upper(), logger_type=main_args.logger
-  )
-
-  config = utils.ConfigBuilder('garf-sql').build(vars(main_args), args[1])
-  logger.debug('config: %s', config)
-  if main_args.save_config and not main_args.garf_config:
-    utils.ConfigSaver(main_args.save_config_dest).save(config)
-  if main_args.dry_run:
-    sys.exit()
-
-  config = utils.initialize_runtime_parameters(config)
-  logger.debug('initialized config: %s', config)
-
-  engine = sqlalchemy.create_engine(config.connection_string)
-  sqlalchemy_query_executor = sql_executor.SqlAlchemyQueryExecutor(engine)
-
-  reader_client = reader.FileReader()
-
-  if main_args.parallel_queries:
-    logger.info('Running queries in parallel')
-    with futures.ThreadPoolExecutor(
-      max_workers=main_args.parallel_threshold
-    ) as executor:
-      future_to_query = {
-        executor.submit(
-          sqlalchemy_query_executor.execute,
-          query,
-          reader_client.read(query),
-          config.params,
-        ): query
-        for query in sorted(main_args.query)
-      }
-      for future in futures.as_completed(future_to_query):
-        query = future_to_query[future]
-        utils.postprocessor_runner(query, future.result, logger)
-  else:
-    logger.info('Running queries sequentially')
-    for query in sorted(main_args.query):
-      callback = functools.partial(
-        executor.execute, query, reader_client.read(query), config.params
-      )
-      utils.postprocessor_runner(query, callback, logger)
-
-
-if __name__ == '__main__':
-  main()

garf_executors-0.0.2.dist-info/METADATA

@@ -1,30 +0,0 @@
-Metadata-Version: 2.1
-Name: garf-executors
-Version: 0.0.2
-Summary: Executes queries against API and writes data to local/remote storage.
-Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>
-License: Apache 2.0
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Intended Audience :: Developers
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Classifier: Operating System :: OS Independent
-Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-Requires-Dist: garf-core
-Requires-Dist: garf-io
-Provides-Extra: bq
-Requires-Dist: garf-io[bq]; extra == "bq"
-Requires-Dist: pandas; extra == "bq"
-Provides-Extra: sql
-Requires-Dist: garf-io[sqlalchemy]; extra == "sql"
-Requires-Dist: pandas; extra == "sql"
-Provides-Extra: all
-Requires-Dist: garf-executors[bq,sql]; extra == "all"
-
-# Gaarf Executors

garf_executors-0.0.2.dist-info/RECORD

@@ -1,16 +0,0 @@
-garf_executors/__init__.py,sha256=EWaHzmbB0XnTnYoR0bNsknBBbCet7tpv7gUwahtBIC0,873
-garf_executors/api_executor.py,sha256=axB7msuZ7LXFk_f3MjIyGM2AhDILyu7g70zqNW0hG6Q,3149
-garf_executors/bq_executor.py,sha256=JBPxbDRYgUgpJv6SqYiFPypTFjZGIZ-SOOb6dS2sZQY,3822
-garf_executors/sql_executor.py,sha256=vBNQ4HZZYxP_EYAh8Z4BerzLESfsNpXdhENzXIw-OIo,2532
-garf_executors/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-garf_executors/entrypoints/utils.py,sha256=C-XbXunnt7HN27eHcTAO7iytu6rM4eWNKApxL345Z6g,15116
-garf_executors/entrypoints/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-garf_executors/entrypoints/cli/api.py,sha256=0QzB7KdXyZnHhaMEGsFhj2B8hyNxV-F5JW6zC90rTcQ,7118
-garf_executors/entrypoints/cli/bq.py,sha256=Rk3nTkcGhyp1hnSSFsLFIFPXo33l_B_O1MiCK2fdZM8,3824
-garf_executors/entrypoints/cli/gaarf.py,sha256=0QzB7KdXyZnHhaMEGsFhj2B8hyNxV-F5JW6zC90rTcQ,7118
-garf_executors/entrypoints/cli/sql.py,sha256=tXQwhrLNUvfORxVdaZHnIawAR06oWZGro2vcreJ22Kc,3753
-garf_executors-0.0.2.dist-info/METADATA,sha256=rxCpgyIWOUo0tQu3HL4kM3cki8eW__tBPgFm3Ee3xIA,1175
-garf_executors-0.0.2.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-garf_executors-0.0.2.dist-info/entry_points.txt,sha256=ksbFBDblKlOYqNyYoL3uaZVYWEYM_KWb0sWrvUamhd4,136
-garf_executors-0.0.2.dist-info/top_level.txt,sha256=sP4dCXOENPn1hDFAunjMV8Js4NND_KGeO_gQWuaT0EY,15
-garf_executors-0.0.2.dist-info/RECORD,,

garf_executors-0.0.2.dist-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-garf-bq-executor = garf_executors.entrypoints.cli.bq:main
-garf-sql-executor = garf_executors.entrypoints.cli.sql:main