garf-core 0.1.5.post0__tar.gz → 0.2.1__tar.gz

{garf_core-0.1.5.post0 → garf_core-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-core
-Version: 0.1.5.post0
+Version: 0.2.1
 Summary: Abstracts fetching data from API based on provided SQL-like query.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

{garf_core-0.1.5.post0 → garf_core-0.2.1}/garf_core/__init__.py

@@ -26,4 +26,4 @@ __all__ = [
   'ApiReportFetcher',
 ]
 
-__version__ = '0.1.5.post0'
+__version__ = '0.2.1'

{garf_core-0.1.5.post0 → garf_core-0.2.1}/garf_core/api_clients.py

@@ -20,23 +20,24 @@ import contextlib
 import csv
 import json
 import os
-import pathlib
 from collections.abc import Sequence
 from typing import Any, Union
 
 import pydantic
 import requests
+import smart_open
 from typing_extensions import TypeAlias, override
 
 from garf_core import exceptions, query_editor
 
 ApiRowElement: TypeAlias = Union[int, float, str, bool, list, dict, None]
+ApiResponseRow: TypeAlias = dict[str, ApiRowElement]
 
 
 class GarfApiResponse(pydantic.BaseModel):
   """Base class for specifying response."""
 
-  results: list[ApiRowElement]
+  results: list[ApiResponseRow]
 
 
 class GarfApiError(exceptions.GarfError):
@@ -123,7 +124,7 @@ class FakeApiClient(BaseClient):
       GarfApiError: When file with data not found.
     """
     try:
-      with pathlib.Path.open(file_location, 'r', encoding='utf-8') as f:
+      with smart_open.open(file_location, 'r', encoding='utf-8') as f:
         data = json.load(f)
         return FakeApiClient(data)
     except FileNotFoundError as e:
@@ -143,7 +144,7 @@ class FakeApiClient(BaseClient):
       GarfApiError: When file with data not found.
     """
     try:
-      with pathlib.Path.open(file_location, 'r', encoding='utf-8') as f:
+      with smart_open.open(file_location, 'r', encoding='utf-8') as f:
         reader = csv.DictReader(f)
         data = []
         for row in reader:

garf_core-0.2.1/garf_core/parsers.py

@@ -0,0 +1,185 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for defining various parsing strategy for API response."""
+
+from __future__ import annotations
+
+import abc
+import ast
+import contextlib
+import functools
+import operator
+from collections.abc import Mapping, MutableSequence
+from typing import Any
+
+from garf_core import api_clients, exceptions, query_editor
+
+VALID_VIRTUAL_COLUMN_OPERATORS = (
+  ast.BinOp,
+  ast.UnaryOp,
+  ast.operator,
+  ast.Constant,
+  ast.Expression,
+)
+
+
+class BaseParser(abc.ABC):
+  """An interface for all parsers to implement."""
+
+  def __init__(
+    self, query_specification: query_editor.BaseQueryElements
+  ) -> None:
+    """Initializes BaseParser."""
+    self.query_spec = query_specification
+
+  def parse_response(
+    self,
+    response: api_clients.GarfApiResponse,
+  ) -> list[list[api_clients.ApiRowElement]]:
+    """Parses response."""
+    if not response.results:
+      return [[]]
+    results = []
+    for result in response.results:
+      results.append(self.parse_row(result))
+    return results
+
+  def _evalute_virtual_column(
+    self,
+    fields: list[str],
+    virtual_column_values: dict[str, Any],
+    substitute_expression: str,
+  ) -> api_clients.ApiRowElement:
+    virtual_column_replacements = {
+      field.replace('.', '_'): value
+      for field, value in zip(fields, virtual_column_values)
+    }
+    virtual_column_expression = substitute_expression.format(
+      **virtual_column_replacements
+    )
+    try:
+      tree = ast.parse(virtual_column_expression, mode='eval')
+      valid = all(
+        isinstance(node, VALID_VIRTUAL_COLUMN_OPERATORS)
+        for node in ast.walk(tree)
+      )
+      if valid:
+        return eval(
+          compile(tree, filename='', mode='eval'), {'__builtins__': None}
+        )
+    except ZeroDivisionError:
+      return 0
+    return None
+
+  def process_virtual_column(
+    self,
+    row: api_clients.ApiResponseRow,
+    virtual_column: query_editor.VirtualColumn,
+  ) -> api_clients.ApiRowElement:
+    if virtual_column.type == 'built-in':
+      return virtual_column.value
+    virtual_column_values = [
+      self.parse_row_element(row, field) for field in virtual_column.fields
+    ]
+    try:
+      result = self._evalute_virtual_column(
+        virtual_column.fields,
+        virtual_column_values,
+        virtual_column.substitute_expression,
+      )
+    except TypeError:
+      virtual_column_values = [
+        f"'{self.parse_row_element(row, field)}'"
+        for field in virtual_column.fields
+      ]
+      result = self._evalute_virtual_column(
+        virtual_column.fields,
+        virtual_column_values,
+        virtual_column.substitute_expression,
+      )
+    except SyntaxError:
+      return virtual_column.value
+    return result
+
+  def parse_row(
+    self,
+    row: api_clients.ApiResponseRow,
+  ) -> list[api_clients.ApiRowElement]:
+    """Parses single row from response."""
+    results = []
+    fields = self.query_spec.fields
+    index = 0
+    for column in self.query_spec.column_names:
+      if virtual_column := self.query_spec.virtual_columns.get(column):
+        result = self.process_virtual_column(row, virtual_column)
+      else:
+        result = self.parse_row_element(row, fields[index])
+        index = index + 1
+      results.append(result)
+    return results
+
+  @abc.abstractmethod
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Returns nested fields from a dictionary."""
+
+
+class DictParser(BaseParser):
+  """Extracts nested dict elements."""
+
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Returns nested fields from a dictionary."""
+    if not isinstance(row, Mapping):
+      raise GarfParserError
+    if result := row.get(key):
+      return result
+    key = key.split('.')
+    try:
+      return functools.reduce(operator.getitem, key, row)
+    except (TypeError, KeyError):
+      return None
+
+
+class NumericConverterDictParser(DictParser):
+  """Extracts nested dict elements with numerical conversions."""
+
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Extract nested field with int/float conversion."""
+
+    def convert_field(value):
+      for type_ in (int, float):
+        with contextlib.suppress(ValueError):
+          return type_(value)
+      return value
+
+    if result := row.get(key):
+      return convert_field(result)
+
+    key = key.split('.')
+    try:
+      field = functools.reduce(operator.getitem, key, row)
+      if isinstance(field, MutableSequence) or field in (True, False):
+        return field
+      return convert_field(field)
+    except KeyError:
+      return None
+
+
+class GarfParserError(exceptions.GarfError):
+  """Incorrect data format for parser."""

{garf_core-0.1.5.post0 → garf_core-0.2.1}/garf_core/query_editor.py

@@ -29,7 +29,7 @@ from typing_extensions import Self, TypeAlias
 
 from garf_core import exceptions
 
-QueryParameters: TypeAlias = dict[str, Union[str, float, int]]
+QueryParameters: TypeAlias = dict[str, Union[str, float, int, list]]
 
 
 class GarfQueryParameters(pydantic.BaseModel):
@@ -125,6 +125,8 @@ class ProcessedField(pydantic.BaseModel):
 
   @classmethod
   def _extract_nested_resource(cls, line_elements: str) -> list[str]:
+    if '://' in line_elements:
+      return []
     return re.split(':', line_elements)
 
 
@@ -160,22 +162,29 @@ class VirtualColumn:
       return VirtualColumn(type='built-in', value=field)
 
     operators = ('/', r'\*', r'\+', ' - ')
-    if len(expressions := re.split('|'.join(operators), field)) > 1:
+    if '://' in field:
+      expressions = re.split(r'\+', field)
+    else:
+      expressions = re.split('|'.join(operators), field)
+    if len(expressions) > 1:
       virtual_column_fields = []
       substitute_expression = field
       for expression in expressions:
         element = expression.strip()
-        if True:
-          # if self._is_valid_field(element):
+        if not _is_constant(element):
           virtual_column_fields.append(element)
           substitute_expression = substitute_expression.replace(
             element, f'{{{element}}}'
           )
+      pattern = r'\{([^}]*)\}'
+      substitute_expression = re.sub(
+        pattern, lambda m: m.group(0).replace('.', '_'), substitute_expression
+      )
       return VirtualColumn(
         type='expression',
         value=field.format(**macros) if macros else field,
         fields=virtual_column_fields,
-        substitute_expression=substitute_expression.replace('.', '_'),
+        substitute_expression=substitute_expression,
       )
     if not _is_quoted_string(field):
       raise GarfFieldError(f"Incorrect field '{field}'.")
@@ -211,7 +220,11 @@ class ExtractedLineElements:
     field, *alias = re.split(' [Aa][Ss] ', line)
     processed_field = ProcessedField.from_raw(field)
     field = processed_field.field
-    virtual_column = None if field else VirtualColumn.from_raw(field, macros)
+    virtual_column = (
+      VirtualColumn.from_raw(field, macros)
+      if _is_invalid_field(field)
+      else None
+    )
     if alias and processed_field.customizer_type:
       customizer = {
         'type': processed_field.customizer_type,
@@ -373,7 +386,7 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
     self,
     text: str,
     title: str | None = None,
-    args: GarfQueryParameters | None = GarfQueryParameters(),
+    args: GarfQueryParameters | None = None,
     **kwargs: str,
   ) -> None:
     """Instantiates QuerySpecification based on text, title and optional args.
@@ -392,7 +405,10 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
     """Returns macros with injected common parameters."""
     common_params = dict(self.common_params)
     if macros := self.args.macro:
-      common_params.update(macros)
+      converted_macros = {
+        key: convert_date(value) for key, value in macros.items()
+      }
+      common_params.update(converted_macros)
     return common_params
 
   def generate(self) -> BaseQueryElements:
@@ -435,10 +451,10 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
       if re.match('/\\*', line) or multiline_comment:
         multiline_comment = True
         continue
-      if re.match('^(#|--|//)', line):
+      if re.match('^(#|--|//) ', line):
         continue
       cleaned_query_line = re.sub(
-        ';$', '', re.sub('(--|//).*$', '', line).strip()
+        ';$', '', re.sub('(--|//) .*$', '', line).strip()
       )
       result.append(cleaned_query_line)
     self.query.text = ' '.join(result)
@@ -506,6 +522,10 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
       line_elements = ExtractedLineElements.from_query_line(line)
       if virtual_column := line_elements.virtual_column:
         self.query.virtual_columns[line_elements.alias] = virtual_column
+        if fields := virtual_column.fields:
+          for field in fields:
+            if field not in self.query.fields:
+              self.query.fields.append(field)
     return self
 
   def extract_customizers(self) -> Self:
@@ -533,3 +553,64 @@ def _is_quoted_string(field_name: str) -> bool:
   return (field_name.startswith("'") and field_name.endswith("'")) or (
     field_name.startswith('"') and field_name.endswith('"')
   )
+
+
+def _is_constant(element) -> bool:
+  with contextlib.suppress(ValueError):
+    float(element)
+    return True
+  return _is_quoted_string(element)
+
+
+def _is_invalid_field(field) -> bool:
+  operators = ('/', '*', '+', ' - ')
+  is_constant = _is_constant(field)
+  has_operator = any(operator in field for operator in operators)
+  return is_constant or has_operator
+
+
+def convert_date(date_string: str) -> str:
+  """Converts specific dates parameters to actual dates.
+
+  Returns:
+    Date string in YYYY-MM-DD format.
+
+  Raises:
+    GarfMacroError:
+     If dynamic lookback value (:YYYYMMDD-N) is incorrect.
+  """
+  if isinstance(date_string, list) or date_string.find(':Y') == -1:
+    return date_string
+  current_date = datetime.date.today()
+  base_date, *date_customizer = re.split('\\+|-', date_string)
+  if len(date_customizer) > 1:
+    raise GarfMacroError(
+      'Invalid format for date macro, should be in :YYYYMMDD-N format'
+    )
+  if not date_customizer:
+    days_lookback = 0
+  else:
+    try:
+      days_lookback = int(date_customizer[0])
+    except ValueError as e:
+      raise GarfMacroError(
+        'Must provide numeric value for a number lookback period, '
+        'i.e. :YYYYMMDD-1'
+      ) from e
+  if base_date == ':YYYY':
+    new_date = datetime.datetime(current_date.year, 1, 1)
+    delta = relativedelta.relativedelta(years=days_lookback)
+  elif base_date == ':YYYYMM':
+    new_date = datetime.datetime(current_date.year, current_date.month, 1)
+    delta = relativedelta.relativedelta(months=days_lookback)
+  elif base_date == ':YYYYMMDD':
+    new_date = current_date
+    delta = relativedelta.relativedelta(days=days_lookback)
+  else:
+    raise GarfMacroError(
+      'Invalid format for date macro, should be in :YYYYMMDD-N format'
+    )
+
+  if '-' in date_string:
+    return (new_date - delta).strftime('%Y-%m-%d')
+  return (new_date + delta).strftime('%Y-%m-%d')

{garf_core-0.1.5.post0 → garf_core-0.2.1}/garf_core/report_fetcher.py

@@ -54,16 +54,18 @@ class ApiReportFetcher:
   """Class responsible for getting data from report API.
 
   Attributes:
-      api_client: a client used for connecting to API.
-      parser: Type of parser to convert API response.
+      api_client: Client used for connecting to API.
+      parser: Class of parser to convert API response.
       query_specification_builder: Class to perform query parsing.
+      builtin_queries:
+        Mapping between query name and function for generating GarfReport.
   """
 
   def __init__(
     self,
-    api_client: api_clients.BaseApiClient,
-    parser: parsers.BaseParser = parsers.DictParser,
-    query_specification_builder: query_editor.QuerySpecification = (
+    api_client: api_clients.BaseClient,
+    parser: type[parsers.BaseParser] = parsers.DictParser,
+    query_specification_builder: type[query_editor.QuerySpecification] = (
       query_editor.QuerySpecification
     ),
     builtin_queries: dict[str, Callable[[ApiReportFetcher], report.GarfReport]]
@@ -94,7 +96,7 @@ class ApiReportFetcher:
 
   async def afetch(
     self,
-    query_specification: str | query_editor.QueryElements,
+    query_specification: str | query_editor.QuerySpecification,
     args: query_editor.GarfQueryParameters | None = None,
     **kwargs: str,
   ) -> report.GarfReport:

{garf_core-0.1.5.post0 → garf_core-0.2.1}/garf_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-core
-Version: 0.1.5.post0
+Version: 0.2.1
 Summary: Abstracts fetching data from API based on provided SQL-like query.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

garf_core-0.1.5.post0/garf_core/parsers.py

@@ -1,119 +0,0 @@
-# Copyright 2025 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Module for defining various parsing strategy for API response."""
-
-from __future__ import annotations
-
-import abc
-import contextlib
-import functools
-import operator
-from collections.abc import Mapping, MutableSequence
-from typing import Any
-
-from typing_extensions import override
-
-from garf_core import api_clients, exceptions, query_editor
-
-
-class BaseParser(abc.ABC):
-  """An interface for all parsers to implement."""
-
-  def __init__(
-    self, query_specification: query_editor.BaseQueryElements
-  ) -> None:
-    """Initializes BaseParser."""
-    self.query_spec = query_specification
-
-  def parse_response(
-    self,
-    response: api_clients.GarfApiResponse,
-  ) -> list[list[api_clients.ApiRowElement]]:
-    """Parses response."""
-    if not response.results:
-      return [[]]
-    results = []
-    for result in response.results:
-      results.append(self.parse_row(result))
-    return results
-
-  @abc.abstractmethod
-  def parse_row(self, row):
-    """Parses single row from response."""
-
-
-class ListParser(BaseParser):
-  """Returns API results as is."""
-
-  @override
-  def parse_row(
-    self,
-    row: list,
-  ) -> list[list[api_clients.ApiRowElement]]:
-    return row
-
-
-class DictParser(BaseParser):
-  """Extracts nested dict elements."""
-
-  @override
-  def parse_row(
-    self,
-    row: list,
-  ) -> list[list[api_clients.ApiRowElement]]:
-    if not isinstance(row, Mapping):
-      raise GarfParserError
-    result = []
-    for field in self.query_spec.fields:
-      result.append(self.get_nested_field(row, field))
-    return result
-
-  def get_nested_field(self, dictionary: dict[str, Any], key: str):
-    """Returns nested fields from a dictionary."""
-    if result := dictionary.get(key):
-      return result
-    key = key.split('.')
-    try:
-      return functools.reduce(operator.getitem, key, dictionary)
-    except (TypeError, KeyError):
-      return None
-
-
-class NumericConverterDictParser(DictParser):
-  """Extracts nested dict elements with numerical conversions."""
-
-  def get_nested_field(self, dictionary: dict[str, Any], key: str):
-    """Extract nested field with int/float conversion."""
-
-    def convert_field(value):
-      for type_ in (int, float):
-        with contextlib.suppress(ValueError):
-          return type_(value)
-      return value
-
-    if result := dictionary.get(key):
-      return convert_field(result)
-
-    key = key.split('.')
-    try:
-      field = functools.reduce(operator.getitem, key, dictionary)
-      if isinstance(field, MutableSequence) or field in (True, False):
-        return field
-      return convert_field(field)
-    except KeyError:
-      return None
-
-
-class GarfParserError(exceptions.GarfError):
-  """Incorrect data format for parser."""