garf-core 0.1.5__tar.gz → 0.2.0__tar.gz

{garf_core-0.1.5 → garf_core-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-core
-Version: 0.1.5
+Version: 0.2.0
 Summary: Abstracts fetching data from API based on provided SQL-like query.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

{garf_core-0.1.5 → garf_core-0.2.0}/garf_core/__init__.py

@@ -26,4 +26,4 @@ __all__ = [
   'ApiReportFetcher',
 ]
 
-__version__ = '0.1.5'
+__version__ = '0.2.0'

{garf_core-0.1.5 → garf_core-0.2.0}/garf_core/api_clients.py

@@ -31,12 +31,13 @@ from typing_extensions import TypeAlias, override
 from garf_core import exceptions, query_editor
 
 ApiRowElement: TypeAlias = Union[int, float, str, bool, list, dict, None]
+ApiResponseRow: TypeAlias = dict[str, ApiRowElement]
 
 
 class GarfApiResponse(pydantic.BaseModel):
   """Base class for specifying response."""
 
-  results: list[ApiRowElement]
+  results: list[ApiResponseRow]
 
 
 class GarfApiError(exceptions.GarfError):
@@ -69,7 +70,7 @@ class RestApiClient(BaseClient):
   ) -> GarfApiResponse:
     response = requests.get(f'{self.endpoint}/{request.resource_name}')
     if response.status_code == self.OK:
-      return GarfApiResponse(response.json())
+      return GarfApiResponse(results=response.json())
     raise GarfApiError('Failed to get data from API')
 
 

garf_core-0.2.0/garf_core/parsers.py

@@ -0,0 +1,185 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for defining various parsing strategy for API response."""
+
+from __future__ import annotations
+
+import abc
+import ast
+import contextlib
+import functools
+import operator
+from collections.abc import Mapping, MutableSequence
+from typing import Any
+
+from garf_core import api_clients, exceptions, query_editor
+
+VALID_VIRTUAL_COLUMN_OPERATORS = (
+  ast.BinOp,
+  ast.UnaryOp,
+  ast.operator,
+  ast.Constant,
+  ast.Expression,
+)
+
+
+class BaseParser(abc.ABC):
+  """An interface for all parsers to implement."""
+
+  def __init__(
+    self, query_specification: query_editor.BaseQueryElements
+  ) -> None:
+    """Initializes BaseParser."""
+    self.query_spec = query_specification
+
+  def parse_response(
+    self,
+    response: api_clients.GarfApiResponse,
+  ) -> list[list[api_clients.ApiRowElement]]:
+    """Parses response."""
+    if not response.results:
+      return [[]]
+    results = []
+    for result in response.results:
+      results.append(self.parse_row(result))
+    return results
+
+  def _evalute_virtual_column(
+    self,
+    fields: list[str],
+    virtual_column_values: dict[str, Any],
+    substitute_expression: str,
+  ) -> api_clients.ApiRowElement:
+    virtual_column_replacements = {
+      field.replace('.', '_'): value
+      for field, value in zip(fields, virtual_column_values)
+    }
+    virtual_column_expression = substitute_expression.format(
+      **virtual_column_replacements
+    )
+    try:
+      tree = ast.parse(virtual_column_expression, mode='eval')
+      valid = all(
+        isinstance(node, VALID_VIRTUAL_COLUMN_OPERATORS)
+        for node in ast.walk(tree)
+      )
+      if valid:
+        return eval(
+          compile(tree, filename='', mode='eval'), {'__builtins__': None}
+        )
+    except ZeroDivisionError:
+      return 0
+    return None
+
+  def process_virtual_column(
+    self,
+    row: api_clients.ApiResponseRow,
+    virtual_column: query_editor.VirtualColumn,
+  ) -> api_clients.ApiRowElement:
+    if virtual_column.type == 'built-in':
+      return virtual_column.value
+    virtual_column_values = [
+      self.parse_row_element(row, field) for field in virtual_column.fields
+    ]
+    try:
+      result = self._evalute_virtual_column(
+        virtual_column.fields,
+        virtual_column_values,
+        virtual_column.substitute_expression,
+      )
+    except TypeError:
+      virtual_column_values = [
+        f"'{self.parse_row_element(row, field)}'"
+        for field in virtual_column.fields
+      ]
+      result = self._evalute_virtual_column(
+        virtual_column.fields,
+        virtual_column_values,
+        virtual_column.substitute_expression,
+      )
+    except SyntaxError:
+      return virtual_column.value
+    return result
+
+  def parse_row(
+    self,
+    row: api_clients.ApiResponseRow,
+  ) -> list[api_clients.ApiRowElement]:
+    """Parses single row from response."""
+    results = []
+    fields = self.query_spec.fields
+    index = 0
+    for column in self.query_spec.column_names:
+      if virtual_column := self.query_spec.virtual_columns.get(column):
+        result = self.process_virtual_column(row, virtual_column)
+      else:
+        result = self.parse_row_element(row, fields[index])
+        index = index + 1
+      results.append(result)
+    return results
+
+  @abc.abstractmethod
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Returns nested fields from a dictionary."""
+
+
+class DictParser(BaseParser):
+  """Extracts nested dict elements."""
+
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Returns nested fields from a dictionary."""
+    if not isinstance(row, Mapping):
+      raise GarfParserError
+    if result := row.get(key):
+      return result
+    key = key.split('.')
+    try:
+      return functools.reduce(operator.getitem, key, row)
+    except (TypeError, KeyError):
+      return None
+
+
+class NumericConverterDictParser(DictParser):
+  """Extracts nested dict elements with numerical conversions."""
+
+  def parse_row_element(
+    self, row: api_clients.ApiResponseRow, key: str
+  ) -> api_clients.ApiRowElement:
+    """Extract nested field with int/float conversion."""
+
+    def convert_field(value):
+      for type_ in (int, float):
+        with contextlib.suppress(ValueError):
+          return type_(value)
+      return value
+
+    if result := row.get(key):
+      return convert_field(result)
+
+    key = key.split('.')
+    try:
+      field = functools.reduce(operator.getitem, key, row)
+      if isinstance(field, MutableSequence) or field in (True, False):
+        return field
+      return convert_field(field)
+    except KeyError:
+      return None
+
+
+class GarfParserError(exceptions.GarfError):
+  """Incorrect data format for parser."""

{garf_core-0.1.5 → garf_core-0.2.0}/garf_core/query_editor.py

@@ -125,6 +125,8 @@ class ProcessedField(pydantic.BaseModel):
 
   @classmethod
   def _extract_nested_resource(cls, line_elements: str) -> list[str]:
+    if '://' in line_elements:
+      return []
     return re.split(':', line_elements)
 
 
@@ -160,22 +162,29 @@ class VirtualColumn:
       return VirtualColumn(type='built-in', value=field)
 
     operators = ('/', r'\*', r'\+', ' - ')
-    if len(expressions := re.split('|'.join(operators), field)) > 1:
+    if '://' in field:
+      expressions = re.split(r'\+', field)
+    else:
+      expressions = re.split('|'.join(operators), field)
+    if len(expressions) > 1:
       virtual_column_fields = []
       substitute_expression = field
       for expression in expressions:
         element = expression.strip()
-        if True:
-          # if self._is_valid_field(element):
+        if not _is_constant(element):
           virtual_column_fields.append(element)
           substitute_expression = substitute_expression.replace(
             element, f'{{{element}}}'
           )
+      pattern = r'\{([^}]*)\}'
+      substitute_expression = re.sub(
+        pattern, lambda m: m.group(0).replace('.', '_'), substitute_expression
+      )
       return VirtualColumn(
         type='expression',
         value=field.format(**macros) if macros else field,
         fields=virtual_column_fields,
-        substitute_expression=substitute_expression.replace('.', '_'),
+        substitute_expression=substitute_expression,
       )
     if not _is_quoted_string(field):
       raise GarfFieldError(f"Incorrect field '{field}'.")
@@ -211,7 +220,11 @@ class ExtractedLineElements:
     field, *alias = re.split(' [Aa][Ss] ', line)
     processed_field = ProcessedField.from_raw(field)
     field = processed_field.field
-    virtual_column = None if field else VirtualColumn.from_raw(field, macros)
+    virtual_column = (
+      VirtualColumn.from_raw(field, macros)
+      if _is_invalid_field(field)
+      else None
+    )
     if alias and processed_field.customizer_type:
       customizer = {
         'type': processed_field.customizer_type,
@@ -435,10 +448,10 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
       if re.match('/\\*', line) or multiline_comment:
         multiline_comment = True
         continue
-      if re.match('^(#|--|//)', line):
+      if re.match('^(#|--|//) ', line):
         continue
       cleaned_query_line = re.sub(
-        ';$', '', re.sub('(--|//).*$', '', line).strip()
+        ';$', '', re.sub('(--|//) .*$', '', line).strip()
       )
       result.append(cleaned_query_line)
     self.query.text = ' '.join(result)
@@ -506,6 +519,10 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
       line_elements = ExtractedLineElements.from_query_line(line)
       if virtual_column := line_elements.virtual_column:
         self.query.virtual_columns[line_elements.alias] = virtual_column
+        if fields := virtual_column.fields:
+          for field in fields:
+            if field not in self.query.fields:
+              self.query.fields.append(field)
     return self
 
   def extract_customizers(self) -> Self:
@@ -533,3 +550,17 @@ def _is_quoted_string(field_name: str) -> bool:
   return (field_name.startswith("'") and field_name.endswith("'")) or (
     field_name.startswith('"') and field_name.endswith('"')
   )
+
+
+def _is_constant(element) -> bool:
+  with contextlib.suppress(ValueError):
+    float(element)
+    return True
+  return _is_quoted_string(element)
+
+
+def _is_invalid_field(field) -> bool:
+  operators = ('/', '*', '+', ' - ')
+  is_constant = _is_constant(field)
+  has_operator = any(operator in field for operator in operators)
+  return is_constant or has_operator

{garf_core-0.1.5 → garf_core-0.2.0}/garf_core/report_fetcher.py

@@ -54,16 +54,18 @@ class ApiReportFetcher:
   """Class responsible for getting data from report API.
 
   Attributes:
-      api_client: a client used for connecting to API.
-      parser: Type of parser to convert API response.
+      api_client: Client used for connecting to API.
+      parser: Class of parser to convert API response.
       query_specification_builder: Class to perform query parsing.
+      builtin_queries:
+        Mapping between query name and function for generating GarfReport.
   """
 
   def __init__(
     self,
-    api_client: api_clients.BaseApiClient,
-    parser: parsers.BaseParser = parsers.DictParser,
-    query_specification_builder: query_editor.QuerySpecification = (
+    api_client: api_clients.BaseClient,
+    parser: type[parsers.BaseParser] = parsers.DictParser,
+    query_specification_builder: type[query_editor.QuerySpecification] = (
       query_editor.QuerySpecification
     ),
     builtin_queries: dict[str, Callable[[ApiReportFetcher], report.GarfReport]]

{garf_core-0.1.5 → garf_core-0.2.0}/garf_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-core
-Version: 0.1.5
+Version: 0.2.0
 Summary: Abstracts fetching data from API based on provided SQL-like query.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

garf_core-0.1.5/garf_core/parsers.py

@@ -1,119 +0,0 @@
-# Copyright 2025 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Module for defining various parsing strategy for API response."""
-
-from __future__ import annotations
-
-import abc
-import contextlib
-import functools
-import operator
-from collections.abc import Mapping, MutableSequence
-from typing import Any
-
-from typing_extensions import override
-
-from garf_core import api_clients, exceptions, query_editor
-
-
-class BaseParser(abc.ABC):
-  """An interface for all parsers to implement."""
-
-  def __init__(
-    self, query_specification: query_editor.BaseQueryElements
-  ) -> None:
-    """Initializes BaseParser."""
-    self.query_spec = query_specification
-
-  def parse_response(
-    self,
-    response: api_clients.GarfApiResponse,
-  ) -> list[list[api_clients.ApiRowElement]]:
-    """Parses response."""
-    if not response.results:
-      return [[]]
-    results = []
-    for result in response.results:
-      results.append(self.parse_row(result))
-    return results
-
-  @abc.abstractmethod
-  def parse_row(self, row):
-    """Parses single row from response."""
-
-
-class ListParser(BaseParser):
-  """Returns API results as is."""
-
-  @override
-  def parse_row(
-    self,
-    row: list,
-  ) -> list[list[api_clients.ApiRowElement]]:
-    return row
-
-
-class DictParser(BaseParser):
-  """Extracts nested dict elements."""
-
-  @override
-  def parse_row(
-    self,
-    row: list,
-  ) -> list[list[api_clients.ApiRowElement]]:
-    if not isinstance(row, Mapping):
-      raise GarfParserError
-    result = []
-    for field in self.query_spec.fields:
-      result.append(self.get_nested_field(row, field))
-    return result
-
-  def get_nested_field(self, dictionary: dict[str, Any], key: str):
-    """Returns nested fields from a dictionary."""
-    if result := dictionary.get(key):
-      return result
-    key = key.split('.')
-    try:
-      return functools.reduce(operator.getitem, key, dictionary)
-    except (TypeError, KeyError):
-      return None
-
-
-class NumericConverterDictParser(DictParser):
-  """Extracts nested dict elements with numerical conversions."""
-
-  def get_nested_field(self, dictionary: dict[str, Any], key: str):
-    """Extract nested field with int/float conversion."""
-
-    def convert_field(value):
-      for type_ in (int, float):
-        with contextlib.suppress(ValueError):
-          return type_(value)
-      return value
-
-    if result := dictionary.get(key):
-      return convert_field(result)
-
-    key = key.split('.')
-    try:
-      field = functools.reduce(operator.getitem, key, dictionary)
-      if isinstance(field, MutableSequence) or field in (True, False):
-        return field
-      return convert_field(field)
-    except KeyError:
-      return None
-
-
-class GarfParserError(exceptions.GarfError):
-  """Incorrect data format for parser."""