garf-core 0.2.2__tar.gz → 0.3.0__tar.gz

{garf_core-0.2.2 → garf_core-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-core
-Version: 0.2.2
+Version: 0.3.0
 Summary: Abstracts fetching data from API based on provided SQL-like query.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

{garf_core-0.2.2 → garf_core-0.3.0}/garf_core/__init__.py

@@ -26,4 +26,4 @@ __all__ = [
   'ApiReportFetcher',
 ]
 
-__version__ = '0.2.2'
+__version__ = '0.3.0'

{garf_core-0.2.2 → garf_core-0.3.0}/garf_core/api_clients.py

@@ -68,10 +68,18 @@ class RestApiClient(BaseClient):
   def get_response(
     self, request: query_editor.BaseQueryElements, **kwargs: str
   ) -> GarfApiResponse:
-    response = requests.get(f'{self.endpoint}/{request.resource_name}')
+    url = f'{self.endpoint}/{request.resource_name}'
+    params = {}
+    for param in request.filters:
+      key, value = param.split('=')
+      params[key.strip()] = value.strip()
+    response = requests.get(url, params=params, headers=kwargs)
     if response.status_code == self.OK:
-      return GarfApiResponse(results=response.json())
-    raise GarfApiError('Failed to get data from API')
+      results = response.json()
+      if not isinstance(results, list):
+        results = [results]
+      return GarfApiResponse(results=results)
+    raise GarfApiError('Failed to get data from API, reason: ', response.text)
 
 
 class FakeApiClient(BaseClient):

{garf_core-0.2.2 → garf_core-0.3.0}/garf_core/parsers.py

@@ -23,7 +23,7 @@ import operator
 from collections.abc import Mapping, MutableSequence
 from typing import Any
 
-from garf_core import api_clients, exceptions, query_editor
+from garf_core import api_clients, exceptions, query_editor, query_parser
 
 VALID_VIRTUAL_COLUMN_OPERATORS = (
   ast.BinOp,
@@ -112,6 +112,20 @@ class BaseParser(abc.ABC):
       return virtual_column.value
     return result
 
+  def process_customizer(
+    self,
+    row: api_clients.ApiResponseRow,
+    customizer: query_parser.Customizer,
+    field: str,
+  ) -> api_clients.ApiRowElement:
+    if customizer.type == 'slice':
+      return self._process_customizer_slice(row, customizer, field)
+    return row
+
+  def _process_customizer_slice(self, row, customizer, field):
+    slice_object = customizer.value.slice_literal
+    return [r.get(customizer.value.value) for r in row.get(field)[slice_object]]
+
   def parse_row(
     self,
     row: api_clients.ApiResponseRow,
@@ -123,6 +137,8 @@ class BaseParser(abc.ABC):
     for column in self.query_spec.column_names:
       if virtual_column := self.query_spec.virtual_columns.get(column):
         result = self.process_virtual_column(row, virtual_column)
+      elif customizer := self.query_spec.customizers.get(column):
+        result = self.process_customizer(row, customizer, fields[index])
       else:
         result = self.parse_row_element(row, fields[index])
         index = index + 1

{garf_core-0.2.2 → garf_core-0.3.0}/garf_core/query_editor.py

@@ -15,7 +15,6 @@
 
 from __future__ import annotations
 
-import contextlib
 import dataclasses
 import datetime
 import logging
@@ -27,7 +26,7 @@ import pydantic
 from dateutil import relativedelta
 from typing_extensions import Self, TypeAlias
 
-from garf_core import exceptions
+from garf_core import query_parser
 
 QueryParameters: TypeAlias = dict[str, Union[str, float, int, list]]
 
@@ -39,219 +38,18 @@ class GarfQueryParameters(pydantic.BaseModel):
   template: QueryParameters = pydantic.Field(default_factory=dict)
 
 
-class GarfQueryError(exceptions.GarfError):
-  """Base exception for Garf queries."""
-
-
-class GarfCustomizerError(GarfQueryError):
-  """Specifies incorrect customizer."""
-
-
-class GarfVirtualColumnError(GarfQueryError):
-  """Specifies incorrect virtual column type."""
-
-
-class GarfFieldError(GarfQueryError):
-  """Specifies incorrect fields from API."""
-
-
-class GarfMacroError(GarfQueryError):
+class GarfMacroError(query_parser.GarfQueryError):
   """Specifies incorrect macro in Garf query."""
 
 
-class GarfResourceError(GarfQueryError):
+class GarfResourceError(query_parser.GarfQueryError):
   """Specifies incorrect resource name in the query."""
 
 
-class GarfBuiltInQueryError(GarfQueryError):
+class GarfBuiltInQueryError(query_parser.GarfQueryError):
   """Specifies non-existing builtin query."""
 
 
-class ProcessedField(pydantic.BaseModel):
-  """Sore field with its customizers.
-
-  Attributes:
-    field: Extractable field.
-    customizer_type: Type of customizer to be applied to the field.
-    customizer_value: Value to be used in customizer.
-  """
-
-  field: str
-  customizer_type: str | None = None
-  customizer_value: int | str | None = None
-
-  @classmethod
-  def from_raw(cls, raw_field: str) -> ProcessedField:
-    """Process field to extract possible customizers.
-
-    Args:
-        raw_field: Unformatted field string value.
-
-    Returns:
-        ProcessedField that contains formatted field with customizers.
-    """
-    raw_field = raw_field.replace(r'\s+', '').strip()
-    if _is_quoted_string(raw_field):
-      return ProcessedField(field=raw_field)
-    if len(resources := cls._extract_resource_element(raw_field)) > 1:
-      field_name, resource_index = resources
-      return ProcessedField(
-        field=field_name,
-        customizer_type='resource_index',
-        customizer_value=int(resource_index),
-      )
-
-    if len(nested_fields := cls._extract_nested_resource(raw_field)) > 1:
-      field_name, nested_field = nested_fields
-      return ProcessedField(
-        field=field_name,
-        customizer_type='nested_field',
-        customizer_value=nested_field,
-      )
-    if len(pointers := cls._extract_pointer(raw_field)) > 1:
-      field_name, pointer = pointers
-      return ProcessedField(
-        field=field_name, customizer_type='pointer', customizer_value=pointer
-      )
-    return ProcessedField(field=raw_field)
-
-  @classmethod
-  def _extract_resource_element(cls, line_elements: str) -> list[str]:
-    return re.split('~', line_elements)
-
-  @classmethod
-  def _extract_pointer(cls, line_elements: str) -> list[str]:
-    return re.split('->', line_elements)
-
-  @classmethod
-  def _extract_nested_resource(cls, line_elements: str) -> list[str]:
-    if '://' in line_elements:
-      return []
-    return re.split(':', line_elements)
-
-
-@dataclasses.dataclass(frozen=True)
-class VirtualColumn:
-  """Represents element in Garf query that either calculated or plugged-in.
-
-  Virtual columns allow performing basic manipulation with metrics and
-  dimensions (i.e. division or multiplication) as well as adding raw text
-  values directly into report.
-
-  Attributes:
-    type: Type of virtual column, either build-in or expression.
-    value: Value of the field after macro expansion.
-    fields: Possible fields participating in calculations.
-    substitute_expression: Formatted expression.
-  """
-
-  type: str
-  value: str
-  fields: list[str] | None = None
-  substitute_expression: str | None = None
-
-  @classmethod
-  def from_raw(cls, field: str, macros: QueryParameters) -> VirtualColumn:
-    """Converts a field to virtual column."""
-    if field.isdigit():
-      field = int(field)
-    else:
-      with contextlib.suppress(ValueError):
-        field = float(field)
-    if isinstance(field, (int, float)):
-      return VirtualColumn(type='built-in', value=field)
-
-    operators = ('/', r'\*', r'\+', ' - ')
-    if '://' in field:
-      expressions = re.split(r'\+', field)
-    else:
-      expressions = re.split('|'.join(operators), field)
-    if len(expressions) > 1:
-      virtual_column_fields = []
-      substitute_expression = field
-      for expression in expressions:
-        element = expression.strip()
-        if not _is_constant(element):
-          virtual_column_fields.append(element)
-          substitute_expression = substitute_expression.replace(
-            element, f'{{{element}}}'
-          )
-      pattern = r'\{([^}]*)\}'
-      substitute_expression = re.sub(
-        pattern, lambda m: m.group(0).replace('.', '_'), substitute_expression
-      )
-      return VirtualColumn(
-        type='expression',
-        value=field.format(**macros) if macros else field,
-        fields=virtual_column_fields,
-        substitute_expression=substitute_expression,
-      )
-    if not _is_quoted_string(field):
-      raise GarfFieldError(f"Incorrect field '{field}'.")
-    field = field.replace("'", '').replace('"', '')
-    field = field.format(**macros) if macros else field
-    return VirtualColumn(type='built-in', value=field)
-
-
-@dataclasses.dataclass
-class ExtractedLineElements:
-  """Helper class for parsing query lines.
-
-  Attributes:
-    fields: All fields extracted from the line.
-    alias: Optional alias assign to a field.
-    virtual_column: Optional virtual column extracted from query line.
-    customizer: Optional values for customizers associated with a field.
-  """
-
-  field: str | None
-  alias: str | None
-  virtual_column: VirtualColumn | None
-  customizer: dict[str, str | int]
-
-  @classmethod
-  def from_query_line(
-    cls,
-    line: str,
-    macros: QueryParameters | None = None,
-  ) -> ExtractedLineElements:
-    if macros is None:
-      macros = {}
-    field, *alias = re.split(' [Aa][Ss] ', line)
-    processed_field = ProcessedField.from_raw(field)
-    field = processed_field.field
-    virtual_column = (
-      VirtualColumn.from_raw(field, macros)
-      if _is_invalid_field(field)
-      else None
-    )
-    if alias and processed_field.customizer_type:
-      customizer = {
-        'type': processed_field.customizer_type,
-        'value': processed_field.customizer_value,
-      }
-    else:
-      customizer = {}
-    if virtual_column and not alias:
-      raise GarfVirtualColumnError('Virtual attributes should be aliased')
-    return ExtractedLineElements(
-      field=_format_type_field_name(field)
-      if not virtual_column and field
-      else None,
-      alias=_normalize_column_name(alias[0] if alias else field),
-      virtual_column=virtual_column,
-      customizer=customizer,
-    )
-
-
-def _format_type_field_name(field_name: str) -> str:
-  return re.sub(r'\.type', '.type_', field_name)
-
-
-def _normalize_column_name(column_name: str) -> str:
-  return re.sub(r'\.', '_', column_name)
-
-
 @dataclasses.dataclass
 class BaseQueryElements:
   """Contains raw query and parsed elements.
@@ -278,7 +76,7 @@ class BaseQueryElements:
   customizers: dict[str, dict[str, str]] = dataclasses.field(
     default_factory=dict
   )
-  virtual_columns: dict[str, VirtualColumn] = dataclasses.field(
+  virtual_columns: dict[str, query_parser.VirtualColumn] = dataclasses.field(
     default_factory=dict
   )
   is_builtin_query: bool = False
@@ -485,7 +283,7 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
 
   def extract_fields(self) -> Self:
     for line in self._extract_query_lines():
-      line_elements = ExtractedLineElements.from_query_line(line)
+      line_elements = query_parser.ExtractedLineElements.from_query_line(line)
       if field := line_elements.field:
         self.query.fields.append(field)
     return self
@@ -514,13 +312,13 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
 
   def extract_column_names(self) -> Self:
     for line in self._extract_query_lines():
-      line_elements = ExtractedLineElements.from_query_line(line)
+      line_elements = query_parser.ExtractedLineElements.from_query_line(line)
       self.query.column_names.append(line_elements.alias)
     return self
 
   def extract_virtual_columns(self) -> Self:
     for line in self._extract_query_lines():
-      line_elements = ExtractedLineElements.from_query_line(line)
+      line_elements = query_parser.ExtractedLineElements.from_query_line(line)
       if virtual_column := line_elements.virtual_column:
         self.query.virtual_columns[line_elements.alias] = virtual_column
         if fields := virtual_column.fields:
@@ -531,7 +329,7 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
 
   def extract_customizers(self) -> Self:
     for line in self._extract_query_lines():
-      line_elements = ExtractedLineElements.from_query_line(line)
+      line_elements = query_parser.ExtractedLineElements.from_query_line(line)
       if customizer := line_elements.customizer:
         self.query.customizers[line_elements.alias] = customizer
     return self
@@ -550,26 +348,6 @@ class QuerySpecification(CommonParametersMixin, TemplateProcessorMixin):
         yield non_empty_row
 
 
-def _is_quoted_string(field_name: str) -> bool:
-  return (field_name.startswith("'") and field_name.endswith("'")) or (
-    field_name.startswith('"') and field_name.endswith('"')
-  )
-
-
-def _is_constant(element) -> bool:
-  with contextlib.suppress(ValueError):
-    float(element)
-    return True
-  return _is_quoted_string(element)
-
-
-def _is_invalid_field(field) -> bool:
-  operators = ('/', '*', '+', ' - ')
-  is_constant = _is_constant(field)
-  has_operator = any(operator in field for operator in operators)
-  return is_constant or has_operator
-
-
 def convert_date(date_string: str) -> str:
   """Converts specific dates parameters to actual dates.
 

garf_core-0.3.0/garf_core/query_parser.py

@@ -0,0 +1,299 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Handles query parsing."""
+
+from __future__ import annotations
+
+import contextlib
+import re
+from typing import Literal, Union
+
+import pydantic
+from typing_extensions import TypeAlias
+
+from garf_core import exceptions
+
+QueryParameters: TypeAlias = dict[str, Union[str, float, int, list]]
+
+CustomizerType: TypeAlias = Literal['resource_index', 'nested_field', 'slice']
+
+
+class GarfQueryError(exceptions.GarfError):
+  """Base exception for Garf queries."""
+
+
+class GarfVirtualColumnError(GarfQueryError):
+  """Specifies incorrect virtual column type."""
+
+
+class GarfCustomizerError(GarfQueryError):
+  """Specifies incorrect customizer."""
+
+
+class GarfFieldError(GarfQueryError):
+  """Specifies incorrect fields from API."""
+
+
+class Customizer(pydantic.BaseModel):
+  """Specifies extraction operation on a field.
+
+  Attributes:
+    type: Type of customizer.
+    value: Value to be extracted from a field.
+  """
+
+  type: CustomizerType | None = None
+  value: int | str | SliceField | None = None
+
+  def __bool__(self) -> bool:
+    """Evaluates whether all fields are not empty."""
+    return bool(self.type and self.value)
+
+
+class SliceField(pydantic.BaseModel):
+  """Specifies slice with the content be extracted.
+
+  Attributes:
+    slice_literal: Slice to be extracted from a sequence.
+    value: Value to be extracted from each element of a slice.
+  """
+
+  model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
+  slice_literal: slice
+  value: str | int
+
+
+class ProcessedField(pydantic.BaseModel):
+  """Stores field with its customizers.
+
+  Attributes:
+    field: Extractable field.
+    customizer: Customizer to be applied to the field.
+  """
+
+  field: str
+  customizer: Customizer = Customizer()
+
+  @classmethod
+  def from_raw(cls, raw_field: str) -> ProcessedField:
+    """Process field to extract possible customizers.
+
+    Args:
+      raw_field: Unformatted field string value.
+
+    Returns:
+      ProcessedField that contains formatted field with customizers.
+    """
+    raw_field = raw_field.replace(r'\s+', '').strip()
+    if _is_quoted_string(raw_field):
+      return ProcessedField(field=raw_field)
+    if len(slices := cls._extract_slices(raw_field)) > 1:
+      field_name, op, slice_literal = slices
+      start, *rest = op.split(':')
+      if start == '':
+        if not rest:
+          slice_object = slice(None)
+        else:
+          end = int(rest[0])
+          slice_object = slice(0, end)
+      elif str.isnumeric(start):
+        if not rest:
+          op_ = int(start)
+          slice_object = slice(op_, op_ + 1)
+        elif rest == ['']:
+          op_ = int(start)
+          slice_object = slice(op_, None)
+        else:
+          op_ = int(start)
+          end = int(rest[0])
+          slice_object = slice(op_, end)
+      return ProcessedField(
+        field=field_name,
+        customizer=Customizer(
+          type='slice',
+          value=SliceField(
+            slice_literal=slice_object, value=re.sub(r'^.', '', slice_literal)
+          ),
+        ),
+      )
+    if len(resources := cls._extract_resource_element(raw_field)) > 1:
+      field_name, resource_index = resources
+      return ProcessedField(
+        field=field_name,
+        customizer=Customizer(type='resource_index', value=int(resource_index)),
+      )
+
+    if len(nested_fields := cls._extract_nested_resource(raw_field)) > 1:
+      field_name, nested_field = nested_fields
+      return ProcessedField(
+        field=field_name,
+        customizer=Customizer(type='nested_field', value=nested_field),
+      )
+    return ProcessedField(field=raw_field)
+
+  @classmethod
+  def _extract_resource_element(cls, line_elements: str) -> list[str]:
+    return re.split('~', line_elements)
+
+  @classmethod
+  def _extract_slices(cls, line_elements: str) -> list[str]:
+    """Finds all slices in the query line."""
+    pattern = r'\[\d*(:\d*)?\]'
+    slices = re.split(pattern, line_elements)
+    regexp = r'\[(\d*(:\d*)?)\]'
+    op = re.findall(regexp, line_elements)
+    if op:
+      slices[1] = op[0][0]
+    return slices
+
+  @classmethod
+  def _extract_nested_resource(cls, line_elements: str) -> list[str]:
+    if '://' in line_elements:
+      return []
+    return re.split(':', line_elements)
+
+
+class VirtualColumn(pydantic.BaseModel):
+  """Represents element in Garf query that either calculated or plugged-in.
+
+  Virtual columns allow performing basic manipulation with metrics and
+  dimensions (i.e. division or multiplication) as well as adding raw text
+  values directly into report.
+
+  Attributes:
+    type: Type of virtual column, either build-in or expression.
+    value: Value of the field after macro expansion.
+    fields: Possible fields participating in calculations.
+    substitute_expression: Formatted expression.
+  """
+
+  type: str
+  value: str | int | float
+  fields: list[str] | None = None
+  substitute_expression: str | None = None
+
+  @classmethod
+  def from_raw(cls, field: str, macros: QueryParameters) -> VirtualColumn:
+    """Converts a field to virtual column."""
+    if field.isdigit():
+      field = int(field)
+    else:
+      with contextlib.suppress(ValueError):
+        field = float(field)
+    if isinstance(field, (int, float)):
+      return VirtualColumn(type='built-in', value=field)
+
+    operators = ('/', r'\*', r'\+', ' - ')
+    if '://' in field:
+      expressions = re.split(r'\+', field)
+    else:
+      expressions = re.split('|'.join(operators), field)
+    if len(expressions) > 1:
+      virtual_column_fields = []
+      substitute_expression = field
+      for expression in expressions:
+        element = expression.strip()
+        if not _is_constant(element):
+          virtual_column_fields.append(element)
+          substitute_expression = substitute_expression.replace(
+            element, f'{{{element}}}'
+          )
+      pattern = r'\{([^}]*)\}'
+      substitute_expression = re.sub(
+        pattern, lambda m: m.group(0).replace('.', '_'), substitute_expression
+      )
+      return VirtualColumn(
+        type='expression',
+        value=field.format(**macros) if macros else field,
+        fields=virtual_column_fields,
+        substitute_expression=substitute_expression,
+      )
+    if not _is_quoted_string(field):
+      raise GarfFieldError(f"Incorrect field '{field}'.")
+    field = field.replace("'", '').replace('"', '')
+    field = field.format(**macros) if macros else field
+    return VirtualColumn(type='built-in', value=field)
+
+
+class ExtractedLineElements(pydantic.BaseModel):
+  """Helper class for parsing query lines.
+
+  Attributes:
+    fields: All fields extracted from the line.
+    alias: Optional alias assign to a field.
+    virtual_column: Optional virtual column extracted from query line.
+    customizer: Optional values for customizers associated with a field.
+  """
+
+  field: str | None
+  alias: str | None
+  virtual_column: VirtualColumn | None
+  customizer: Customizer | None
+
+  @classmethod
+  def from_query_line(
+    cls,
+    line: str,
+    macros: QueryParameters | None = None,
+  ) -> ExtractedLineElements:
+    if macros is None:
+      macros = {}
+    field, *alias = re.split(' [Aa][Ss] ', line)
+    processed_field = ProcessedField.from_raw(field)
+    field = processed_field.field
+    virtual_column = (
+      VirtualColumn.from_raw(field, macros)
+      if _is_invalid_field(field)
+      else None
+    )
+    if not (customizer := processed_field.customizer):
+      customizer = None
+    if virtual_column and not alias:
+      raise GarfVirtualColumnError('Virtual attributes should be aliased')
+    return ExtractedLineElements(
+      field=_format_type_field_name(field)
+      if not virtual_column and field
+      else None,
+      alias=_normalize_column_name(alias[0] if alias else field),
+      virtual_column=virtual_column,
+      customizer=customizer,
+    )
+
+
+def _format_type_field_name(field_name: str) -> str:
+  return re.sub(r'\.type', '.type_', field_name)
+
+
+def _normalize_column_name(column_name: str) -> str:
+  return re.sub(r'\.', '_', column_name)
+
+
+def _is_quoted_string(field_name: str) -> bool:
+  return (field_name.startswith("'") and field_name.endswith("'")) or (
+    field_name.startswith('"') and field_name.endswith('"')
+  )
+
+
+def _is_constant(element) -> bool:
+  with contextlib.suppress(ValueError):
+    float(element)
+    return True
+  return _is_quoted_string(element)
+
+
+def _is_invalid_field(field) -> bool:
+  operators = ('/', '*', '+', ' - ')
+  is_constant = _is_constant(field)
+  has_operator = any(operator in field for operator in operators)
+  return is_constant or has_operator

{garf_core-0.2.2 → garf_core-0.3.0}/garf_core/report_fetcher.py

@@ -150,5 +150,7 @@ class ApiReportFetcher:
     response = self.api_client.get_response(query, **kwargs)
     parsed_response = self.parser(query).parse_response(response)
     return report.GarfReport(
-      results=parsed_response, column_names=query.column_names
+      results=parsed_response,
+      column_names=query.column_names,
+      query_specification=query,
     )

{garf_core-0.2.2 → garf_core-0.3.0}/garf_core.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: garf-core
-Version: 0.2.2
+Version: 0.3.0
 Summary: Abstracts fetching data from API based on provided SQL-like query.
 Author-email: "Google Inc. (gTech gPS CSE team)" <no-reply@google.com>, Andrei Markin <andrey.markin.ppc@gmail.com>
 License: Apache 2.0

{garf_core-0.2.2 → garf_core-0.3.0}/garf_core.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ garf_core/base_query.py
 garf_core/exceptions.py
 garf_core/parsers.py
 garf_core/query_editor.py
+garf_core/query_parser.py
 garf_core/report.py
 garf_core/report_fetcher.py
 garf_core.egg-info/PKG-INFO