keboola.component 1.6.6__tar.gz → 1.6.8__tar.gz

{keboola.component-1.6.6/src/keboola.component.egg-info → keboola.component-1.6.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: keboola.component
-Version: 1.6.6
+Version: 1.6.8
 Summary: General library for Python applications running in Keboola Connection environment
 Home-page: https://github.com/keboola/python-component
 Author: Keboola KDS Team

{keboola.component-1.6.6 → keboola.component-1.6.8}/setup.py

@@ -13,7 +13,7 @@ project_urls = {
 
 setuptools.setup(
     name="keboola.component",
-    version="1.6.6",
+    version="1.6.8",
     author="Keboola KDS Team",
     project_urls=project_urls,
     setup_requires=['pytest-runner', 'flake8'],

{keboola.component-1.6.6 → keboola.component-1.6.8}/src/keboola/component/base.py

@@ -249,6 +249,18 @@ class ComponentBase(ABC, CommonInterface):
             raise AttributeError(f"The defined action {action} is not implemented!") from e
         return action_method()
 
+    def _generate_table_metadata_legacy(self, table_schema: ts.TableSchema) -> dao.TableMetadata:
+        """
+            Generates a TableMetadata object for the table definition using a TableSchema object.
+
+        """
+        table_metadata = dao.TableMetadata()
+        if table_schema.description:
+            table_metadata.add_table_description(table_schema.description)
+        table_metadata.add_column_descriptions({field.name: field.description for field in table_schema.fields})
+        table_metadata = self._add_field_data_types_to_table_metadata(table_schema, table_metadata)
+        return table_metadata
+
     def create_out_table_definition_from_schema(self, table_schema: ts.TableSchema, is_sliced: bool = False,
                                                 destination: str = '', incremental: bool = None,
                                                 enclosure: str = '"', delimiter: str = ',',
@@ -271,17 +283,33 @@ class ComponentBase(ABC, CommonInterface):
                 TableDefinition object initialized with all table metadata defined in a schema
 
         """
-        table_metadata = self._generate_table_metadata(table_schema)
-        return self.create_out_table_definition(name=table_schema.csv_name,
-                                                columns=table_schema.field_names,
-                                                primary_key=table_schema.primary_keys,
-                                                table_metadata=table_metadata,
-                                                is_sliced=is_sliced,
-                                                destination=destination,
-                                                incremental=incremental,
-                                                enclosure=enclosure,
-                                                delimiter=delimiter,
-                                                delete_where=delete_where)
+        if self._expects_legacy_manifest():
+            table_metadata = self._generate_table_metadata_legacy(table_schema)
+            table_def = self.create_out_table_definition(name=table_schema.csv_name,
+                                                         columns=table_schema.field_names,
+                                                         primary_key=table_schema.primary_keys,
+                                                         table_metadata=table_metadata,
+                                                         is_sliced=is_sliced,
+                                                         destination=destination,
+                                                         incremental=incremental,
+                                                         enclosure=enclosure,
+                                                         delimiter=delimiter,
+                                                         delete_where=delete_where)
+        else:
+            schema = self._generate_schema_definition(table_schema)
+
+            table_def = self.create_out_table_definition(name=table_schema.csv_name,
+                                                         primary_key=table_schema.primary_keys,
+                                                         schema=schema,
+                                                         is_sliced=is_sliced,
+                                                         destination=destination,
+                                                         incremental=incremental,
+                                                         enclosure=enclosure,
+                                                         delimiter=delimiter,
+                                                         delete_where=delete_where,
+                                                         description=table_schema.description)
+
+        return table_def
 
     def get_table_schema_by_name(self, schema_name: str,
                                  schema_folder_path: Optional[str] = None) -> ts.TableSchema:
@@ -324,18 +352,24 @@ class ComponentBase(ABC, CommonInterface):
                                     "from a schema. If a schema folder path is not defined, the schemas folder must be"
                                     " located in the 'src' directory of a component : src/schemas")
 
-    def _generate_table_metadata(self, table_schema: ts.TableSchema) -> dao.TableMetadata:
+    def _generate_schema_definition(self, table_schema: ts.TableSchema) -> Dict[str, dao.ColumnDefinition]:
         """
             Generates a TableMetadata object for the table definition using a TableSchema object.
 
         """
-        table_metadata = dao.TableMetadata()
-        if table_schema.description:
-            table_metadata.add_table_description(table_schema.description)
-        table_metadata.add_column_descriptions({field.name: field.description for field in table_schema.fields})
-        table_metadata = self._add_field_data_types_to_table_metadata(table_schema, table_metadata)
+        column_definitions = {}
+        for field in table_schema.fields:
+            if field.base_type:
+                data_types = dao.BaseType(field.base_type,
+                                          length=field.length,
+                                          default=field.default)
+            else:
+                data_types = dao.BaseType()
+            column_definitions[field.name] = dao.ColumnDefinition(data_types=data_types,
+                                                                  nullable=field.nullable,
+                                                                  description=field.description)
 
-        return table_metadata
+        return column_definitions
 
     @staticmethod
     def _add_field_data_types_to_table_metadata(table_schema: ts.TableSchema,

{keboola.component-1.6.6 → keboola.component-1.6.8}/src/keboola/component/dao.py

@@ -1,5 +1,6 @@
 # Python 3.7 support
 from __future__ import annotations
+
 import dataclasses
 import json
 import logging
@@ -12,7 +13,6 @@ from enum import Enum
 from pathlib import Path
 from typing import List, Union, Dict, Optional, OrderedDict as TypeOrderedDict
 
-
 from deprecated import deprecated
 
 from .exceptions import UserException
@@ -66,6 +66,7 @@ class EnvironmentVariables:
     logger_port: str
 
     data_type_support: str
+    project_features: str
 
 
 class SupportedDataTypes(str, Enum):
@@ -203,15 +204,15 @@ class TableMetadata:
         """
         if legacy_manifest:
             final_metadata_list = [{'key': key,
-                                    'value': self.table_metadata[key]}
-                                   for key in self.table_metadata]
+                                    'value': value}
+                                   for key, value in self.table_metadata.items() if value not in [None, '']]
         else:
-            final_metadata_list = [{key: self.table_metadata[key]}
-                                   for key in self.table_metadata]
+            final_metadata_list = {key: value
+                                   for key, value in self.table_metadata.items() if value not in [None, '']}
 
         return final_metadata_list
 
-    @deprecated(version='1.5.1', reason="Please use schema instead of Table Metadata")
+    @deprecated(version='1.5.1', reason="Please use schema instead of Column Metadata")
     def get_column_metadata_for_manifest(self) -> dict:
         """
                 Returns column metadata dict as required by the
@@ -250,13 +251,14 @@ class TableMetadata:
                 final_column_metadata[column] = list()
 
             column_metadata = [{'key': key,
-                                'value': column_metadata_dicts[key]} for key in
-                               column_metadata_dicts]
+                                'value': value} for key, value in
+                               column_metadata_dicts.items() if value not in [None, '']]
             final_column_metadata[column].extend(column_metadata)
 
         return final_column_metadata
 
     @property
+    @deprecated(version='1.5.1', reason="Please use TableDefinition.description instead of TableMetadata")
     def table_description(self) -> str:
         """
         Returns table description (KBC.description)
@@ -401,14 +403,20 @@ class TableMetadata:
                 Args:
 
         """
+        if value is None:
+            return
         self.table_metadata = {**self.table_metadata, **{key: value}}
 
+    @deprecated(version='1.5.1', reason="Column metadata ere moved to dao.TableDefinition.schema property."
+                                        "Please use the dao.ColumnDefinition.metadata")
     def add_column_metadata(self, column: str, key: str, value: Union[str, bool, int], backend="base"):
         """
         Add/Updates column metadata and ensures the Key is unique.
         Args:
 
         """
+        if value is None:
+            return
         if not self.column_metadata.get(column):
             self.column_metadata[column] = dict()
 
@@ -416,6 +424,8 @@ class TableMetadata:
 
         # self.schema = [ColumnDefinition(name=column, data_type={backend: DataType(type=value)})]
 
+    @deprecated(version='1.5.1', reason="Column metadata ere moved to dao.TableDefinition.schema property."
+                                        "Please use the dao.ColumnDefinition.metadata")
     def add_multiple_column_metadata(self, column_metadata: Dict[str, List[dict]]):
         """
         Add key-value pairs to column metadata.
@@ -576,7 +586,7 @@ class SupportedManifestAttributes(SubscriptableDataclass):
                 to_remove = ['primary_key', 'columns', 'distribution_key', 'column_metadata', 'metadata']
                 attributes = list(set(attributes).difference(to_remove))
 
-                to_add = ['manifest_type', 'has_header', 'description', 'table_metadata', 'schema']
+                to_add = ['manifest_type', 'has_header', 'table_metadata', 'schema']
                 attributes.extend(to_add)
 
         elif stage == 'in':
@@ -772,13 +782,14 @@ class TableDefinition(IODefinition):
 
     MANIFEST_ATTRIBUTES = {'in': INPUT_MANIFEST_ATTRIBUTES,
                            'out': OUTPUT_MANIFEST_ATTRIBUTES}
+    SCHEMA_TYPE = Union[Dict[str, ColumnDefinition], TypeOrderedDict[str, ColumnDefinition], List[str]]
 
     def __init__(self, name: str,
                  full_path: Optional[Union[str, None]] = None,
                  is_sliced: Optional[bool] = False,
                  destination: Optional[str] = '',
                  primary_key: Optional[List[str]] = None,
-                 schema: Optional[Union[TypeOrderedDict[str, ColumnDefinition], list[str]]] = None,
+                 schema: SCHEMA_TYPE = None,
                  incremental: Optional[bool] = None,
                  table_metadata: Optional[TableMetadata] = None,
                  enclosure: Optional[str] = '"',
@@ -787,6 +798,7 @@ class TableDefinition(IODefinition):
                  stage: Optional[str] = 'out',
                  write_always: Optional[bool] = False,
                  has_header: Optional[bool] = None,
+                 description: Optional[str] = None,
                  # input
                  **kwargs
                  ):
@@ -814,6 +826,7 @@ class TableDefinition(IODefinition):
             write_always: Bool: If true, the table will be saved to Storage even when the job execution
                            fails.
             schema: (dict|lis[str]) Mapping of column names andColumnDefinition objects, or a list of names
+            description: str: Table description
 
         """
         super().__init__(full_path)
@@ -823,7 +836,7 @@ class TableDefinition(IODefinition):
         # initialize manifest properties
         self._destination = None
         self.destination = destination
-        self._schema = dict()
+        self._schema: Dict[str, ColumnDefinition] = dict()
 
         if schema:
             self.schema = schema
@@ -844,6 +857,8 @@ class TableDefinition(IODefinition):
         if not table_metadata:
             table_metadata = TableMetadata()
         self.table_metadata = table_metadata
+        if description:
+            self.table_metadata.add_table_description(description)
 
         self.delete_where_values = None
         self.delete_where_column = None
@@ -892,7 +907,8 @@ class TableDefinition(IODefinition):
                                 delimiter: Optional[str] = ',',
                                 delete_where: Optional[dict] = None,
                                 write_always: Optional[bool] = False,
-                                schema: Optional[Union[TypeOrderedDict[str, ColumnDefinition], list[str]]] = None,
+                                schema: SCHEMA_TYPE = None,
+                                description: Optional[str] = None,
                                 **kwargs
                                 ):
         """
@@ -913,6 +929,7 @@ class TableDefinition(IODefinition):
             delete_where (Optional[dict]): Criteria for row deletion in incremental loads. Defaults to None.
             write_always (Optional[bool]): If True, the table will be saved to storage even if the job fails.
             schema (Optional[List[ColumnDefinition]]): Dictionary of ColumnDefinition objects.
+            description (Optional[str]): The description of the table. Defaults to None.
 
         Returns:
             TableDefinition: An instance of TableDefinition configured for output tables.
@@ -928,6 +945,7 @@ class TableDefinition(IODefinition):
                    delete_where=delete_where,
                    write_always=write_always,
                    schema=schema,
+                   description=description,
                    **kwargs
                    )
 
@@ -1154,7 +1172,7 @@ class TableDefinition(IODefinition):
                 name=name,
                 destination=manifest.get('destination'),
                 schema=cls.return_schema_from_manifest(manifest),
-                incremental=manifest.get('incremental'),
+                incremental=manifest.get('incremental', False),
                 primary_key=manifest.get('primary_key'),
                 write_always=manifest.get('write_always', False),
                 delimiter=manifest.get('delimiter', ','),
@@ -1254,7 +1272,6 @@ class TableDefinition(IODefinition):
             'column_metadata': self.table_metadata._get_legacy_column_metadata_for_manifest(),
             'manifest_type': manifest_type,
             'has_header': self.has_header,
-            'description': None,
             'table_metadata': self.table_metadata.get_table_metadata_for_manifest(),
             'delete_where_column': self.delete_where_column,
             'delete_where_values': self.delete_where_values,
@@ -1262,6 +1279,7 @@ class TableDefinition(IODefinition):
             'schema': [col.to_dict(name)
                        for name, col in self.schema.items()] if isinstance(self.schema, (OrderedDict, dict)) else []
         }
+
         if legacy_manifest or self.stage == 'in':
             fields['columns'] = self.column_names
 
@@ -1386,8 +1404,8 @@ class TableDefinition(IODefinition):
 
     @incremental.setter
     def incremental(self, incremental: bool):
-        if incremental:
-            self._incremental = True
+        if incremental is not None:
+            self._incremental = incremental
 
     @property
     def write_always(self) -> bool:

{keboola.component-1.6.6 → keboola.component-1.6.8}/src/keboola/component/interface.py

@@ -18,7 +18,7 @@ from pygelf import GelfUdpHandler, GelfTcpHandler
 from pytz import utc
 
 from . import dao
-from .dao import ColumnDefinition
+from .dao import ColumnDefinition, TableDefinition
 from .exceptions import UserException
 
 
@@ -55,7 +55,8 @@ def init_environment_variables() -> dao.EnvironmentVariables:
                                     real_user=os.environ.get('KBC_REALUSER', None),
                                     logger_addr=os.environ.get('KBC_LOGGER_ADDR', None),
                                     logger_port=os.environ.get('KBC_LOGGER_PORT', None),
-                                    data_type_support=os.environ.get('KBC_DATA_TYPE_SUPPORT', None)
+                                    data_type_support=os.environ.get('KBC_DATA_TYPE_SUPPORT', None),
+                                    project_features=os.environ.get('KBC_PROJECT_FEATURE_GATES', '')
                                     )
 
 
@@ -337,7 +338,8 @@ class CommonInterface:
                                  write_always: bool = False,
                                  schema: Union[
                                      OrderedDict[str, ColumnDefinition], list[str]] = None,
-                                 has_header: Optional[bool] = None) -> dao.TableDefinition:
+                                 has_header: Optional[bool] = None,
+                                 description: Optional[str] = None) -> dao.TableDefinition:
         """
                 Helper method for dao.TableDefinition creation along with the "manifest".
                 It initializes path according to the storage_stage type.
@@ -380,6 +382,7 @@ class CommonInterface:
                                    primary_key=primary_key,
                                    columns=columns,
                                    incremental=incremental,
+                                   description=description,
                                    table_metadata=table_metadata,
                                    enclosure=enclosure,
                                    delimiter=delimiter,
@@ -426,13 +429,11 @@ class CommonInterface:
                                              delete_where=delete_where,
                                              schema=schema)
 
-    SCHEMA_TYPE = Union[Dict[str, ColumnDefinition], OrderedDict[str, ColumnDefinition], List[str]]
-
     def create_out_table_definition(self, name: str,
                                     is_sliced: bool = False,
                                     destination: str = '',
                                     primary_key: List[str] = None,
-                                    schema: SCHEMA_TYPE = None,
+                                    schema: TableDefinition.SCHEMA_TYPE = None,
                                     incremental: bool = None,
                                     table_metadata: dao.TableMetadata = None,
                                     enclosure: str = '"',
@@ -440,6 +441,7 @@ class CommonInterface:
                                     delete_where: dict = None,
                                     write_always: bool = False,
                                     has_header: Optional[bool] = None,
+                                    description: Optional[str] = None,
                                     **kwargs
                                     ) -> dao.TableDefinition:
         """
@@ -463,6 +465,7 @@ class CommonInterface:
                            fails.
                            has_header:Optional[bool] = flag whether the header is present in the file,
                                 if None legacy method is used
+                           description: Table description
         """
 
         return self._create_table_definition(name=name,
@@ -478,7 +481,8 @@ class CommonInterface:
                                              delete_where=delete_where,
                                              write_always=write_always,
                                              schema=schema,
-                                             has_header=has_header)
+                                             has_header=has_header,
+                                             description=description)
 
     # # File processing
 
@@ -902,6 +906,10 @@ class CommonInterface:
     def files_in_path(self):
         return os.path.join(self.data_folder_path, 'in', 'files')
 
+    @property
+    def _running_in_kbc(self):
+        return self.environment_variables.stack_id or False
+
     @property
     def is_legacy_queue(self) -> bool:
         """
@@ -909,9 +917,10 @@ class CommonInterface:
         Returns:
 
         """
-        features = os.environ.get('KBC_PROJECT_FEATURE_GATES')
+        features = self.environment_variables.project_features
+
         is_legacy_queue = True
-        if not features or 'queuev2' in features:
+        if not self._running_in_kbc or 'queuev2' in features:
             is_legacy_queue = False
         return is_legacy_queue
 
@@ -950,7 +959,6 @@ class CommonInterface:
         """
 
         if not legacy_manifest:
-
             legacy_manifest = self._expects_legacy_manifest()
 
         manifest = io_definition.get_manifest_dictionary(legacy_queue=self.is_legacy_queue,
@@ -961,7 +969,9 @@ class CommonInterface:
             json.dump(manifest, manifest_file)
 
     def _expects_legacy_manifest(self) -> bool:
-        legacy_manifest = self.environment_variables.data_type_support not in ('authoritative', 'hints')
+        legacy_manifest = \
+            (self._running_in_kbc and self.environment_variables.data_type_support not in ('authoritative', 'hints'))
+
         om_override = self.configuration.config_data.get('storage', {}).get('output', {}).get('data_type_support')
         if om_override:
             legacy_manifest = om_override not in ('authoritative', 'hints')

{keboola.component-1.6.6 → keboola.component-1.6.8/src/keboola.component.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: keboola.component
-Version: 1.6.6
+Version: 1.6.8
 Summary: General library for Python applications running in Keboola Connection environment
 Home-page: https://github.com/keboola/python-component
 Author: Keboola KDS Team