keboola.component 1.6.3__tar.gz → 1.6.4__tar.gz

{keboola.component-1.6.3/src/keboola.component.egg-info → keboola.component-1.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: keboola.component
-Version: 1.6.3
+Version: 1.6.4
 Summary: General library for Python applications running in Keboola Connection environment
 Home-page: https://github.com/keboola/python-component
 Author: Keboola KDS Team
@@ -231,6 +231,59 @@ possible to create the container for the output table using the `CommonInterface
 
 ![TableDefinition dependencies](docs/imgs/TableDefinition_class.png)
 
+**Table schema example:**
+
+```python
+from keboola.component import CommonInterface
+from keboola.component.dao import ColumnDefinition, DataType, SupportedDataTypes, BaseType
+
+# init the interface
+ci = CommonInterface(data_folder_path='data')
+
+# create container for the result
+out = ci.create_out_table_definition("testDef",
+                                     columns=['foo', 'bar'],
+                                     destination='some-destination',
+                                     primary_key=['foo'],
+                                     incremental=True,
+                                     delete_where={'column': 'lilly',
+                                                   'values': ['a', 'b'],
+                                                   'operator': 'eq'})
+
+# update column
+out.update_column('foo',
+                  ColumnDefinition(data_types=BaseType(dtype=SupportedDataTypes.INTEGER, length='20')))
+
+# add new columns
+out.add_column('note', ColumnDefinition(nullable=False))
+out.add_column('test1')
+out.add_columns(['test2', 'test3', 'test4'])
+
+# add new typed column
+out.add_column('id', ColumnDefinition(primary_key=True,
+                                      data_types={'snowflake': DataType(dtype="INTEGER", length='200')})
+               )
+
+out.add_columns({
+    'phone': ColumnDefinition(primary_key=True,
+                              data_types={'snowflake': DataType(dtype="INTEGER", length='200'),
+                                          'bigquery': DataType(dtype="BIGINT")}),
+    'new2': ColumnDefinition(data_types={'snowflake': DataType(dtype="INTEGER", length='200')}),
+                 })
+
+# delete columns
+out.delete_column('bar')
+out.delete_columns(['test2', 'test3'])
+
+
+# write some content
+with open(out.full_path, 'w') as result:
+    result.write('line')
+    
+# write manifest
+ci.write_manifest(out)
+```
+
 **Example:**
 
 ```python

{keboola.component-1.6.3 → keboola.component-1.6.4}/README.md

@@ -212,6 +212,59 @@ possible to create the container for the output table using the `CommonInterface
 
 ![TableDefinition dependencies](docs/imgs/TableDefinition_class.png)
 
+**Table schema example:**
+
+```python
+from keboola.component import CommonInterface
+from keboola.component.dao import ColumnDefinition, DataType, SupportedDataTypes, BaseType
+
+# init the interface
+ci = CommonInterface(data_folder_path='data')
+
+# create container for the result
+out = ci.create_out_table_definition("testDef",
+                                     columns=['foo', 'bar'],
+                                     destination='some-destination',
+                                     primary_key=['foo'],
+                                     incremental=True,
+                                     delete_where={'column': 'lilly',
+                                                   'values': ['a', 'b'],
+                                                   'operator': 'eq'})
+
+# update column
+out.update_column('foo',
+                  ColumnDefinition(data_types=BaseType(dtype=SupportedDataTypes.INTEGER, length='20')))
+
+# add new columns
+out.add_column('note', ColumnDefinition(nullable=False))
+out.add_column('test1')
+out.add_columns(['test2', 'test3', 'test4'])
+
+# add new typed column
+out.add_column('id', ColumnDefinition(primary_key=True,
+                                      data_types={'snowflake': DataType(dtype="INTEGER", length='200')})
+               )
+
+out.add_columns({
+    'phone': ColumnDefinition(primary_key=True,
+                              data_types={'snowflake': DataType(dtype="INTEGER", length='200'),
+                                          'bigquery': DataType(dtype="BIGINT")}),
+    'new2': ColumnDefinition(data_types={'snowflake': DataType(dtype="INTEGER", length='200')}),
+                 })
+
+# delete columns
+out.delete_column('bar')
+out.delete_columns(['test2', 'test3'])
+
+
+# write some content
+with open(out.full_path, 'w') as result:
+    result.write('line')
+    
+# write manifest
+ci.write_manifest(out)
+```
+
 **Example:**
 
 ```python

{keboola.component-1.6.3 → keboola.component-1.6.4}/setup.py

@@ -13,7 +13,7 @@ project_urls = {
 
 setuptools.setup(
     name="keboola.component",
-    version="1.6.3",
+    version="1.6.4",
     author="Keboola KDS Team",
     project_urls=project_urls,
     setup_requires=['pytest-runner', 'flake8'],

{keboola.component-1.6.3 → keboola.component-1.6.4}/src/keboola/component/dao.py

@@ -582,6 +582,7 @@ class SupportedManifestAttributes(SubscriptableDataclass):
         elif stage == 'in':
             attributes = self.in_attributes
             exclude = self.in_legacy_exclude
+
         else:
             raise ValueError(f'Unsupported stage {stage}')
 
@@ -743,6 +744,11 @@ class TableDefinition(IODefinition):
         "columns",
         "metadata",
         "column_metadata",
+        "rows_count",
+        "data_size_bytes",
+        "is_alias",
+        "attributes",
+        "indexed_columns"
     ]
 
     OUTPUT_MANIFEST_ATTRIBUTES = [
@@ -855,6 +861,8 @@ class TableDefinition(IODefinition):
         self._rows_count = kwargs.get('rows_count')
         self._data_size_bytes = kwargs.get('data_size_bytes')
         self._is_alias = kwargs.get('is_alias')
+        self._indexed_columns = kwargs.get('indexed_columns')
+        self._attributes = kwargs.get('attributes')
 
         self.stage = stage
         self.has_header = has_header or self._has_header_in_file()
@@ -1107,12 +1115,14 @@ class TableDefinition(IODefinition):
                         rows_count=manifest.get('rows_count'),
                         data_size_bytes=manifest.get('data_size_bytes'),
                         is_alias=manifest.get('is_alias'),
-                        force_legacy_mode=force_legacy_mode
+                        force_legacy_mode=force_legacy_mode,
+                        indexed_columns=manifest.get('indexed_columns'),
+                        attributes=manifest.get('attributes')
                         )
 
         return table_def
 
-    def get_manifest_dictionary(self, manifest_type: Optional[str] = 'out', legacy_queue: bool = False,
+    def get_manifest_dictionary(self, manifest_type: Optional[str] = None, legacy_queue: bool = False,
                                 legacy_manifest: Optional[bool] = None) -> dict:
         """
         Returns manifest dictionary in appropriate manifest_type: either 'in' or 'out'.
@@ -1184,6 +1194,8 @@ class TableDefinition(IODefinition):
             'rows_count': self._rows_count,
             'data_size_bytes': self._data_size_bytes,
             'is_alias': self._is_alias,
+            'indexed_columns': self._indexed_columns,
+            'attributes': self._attributes,
 
             'destination': self.destination,
             'incremental': self.incremental,

{keboola.component-1.6.3 → keboola.component-1.6.4}/src/keboola/component/interface.py

@@ -950,7 +950,8 @@ class CommonInterface:
         """
 
         if not legacy_manifest:
-            legacy_manifest = self.environment_variables.data_type_support not in ('authoritative', 'hints')
+
+            legacy_manifest = self._expects_legacy_manifest()
 
         manifest = io_definition.get_manifest_dictionary(legacy_queue=self.is_legacy_queue,
                                                          legacy_manifest=legacy_manifest)
@@ -959,6 +960,13 @@ class CommonInterface:
         with open(io_definition.full_path + '.manifest', 'w') as manifest_file:
             json.dump(manifest, manifest_file)
 
+    def _expects_legacy_manifest(self) -> bool:
+        legacy_manifest = self.environment_variables.data_type_support not in ('authoritative', 'hints')
+        om_override = self.configuration.config_data.get('storage', {}).get('output', {}).get('data_type_support')
+        if om_override:
+            legacy_manifest = om_override not in ('authoritative', 'hints')
+        return legacy_manifest
+
     def write_manifests(self, io_definitions: List[Union[dao.FileDefinition, dao.TableDefinition]],
                         legacy_manifest: Optional[bool] = None):
         """

{keboola.component-1.6.3 → keboola.component-1.6.4/src/keboola.component.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: keboola.component
-Version: 1.6.3
+Version: 1.6.4
 Summary: General library for Python applications running in Keboola Connection environment
 Home-page: https://github.com/keboola/python-component
 Author: Keboola KDS Team
@@ -231,6 +231,59 @@ possible to create the container for the output table using the `CommonInterface
 
 ![TableDefinition dependencies](docs/imgs/TableDefinition_class.png)
 
+**Table schema example:**
+
+```python
+from keboola.component import CommonInterface
+from keboola.component.dao import ColumnDefinition, DataType, SupportedDataTypes, BaseType
+
+# init the interface
+ci = CommonInterface(data_folder_path='data')
+
+# create container for the result
+out = ci.create_out_table_definition("testDef",
+                                     columns=['foo', 'bar'],
+                                     destination='some-destination',
+                                     primary_key=['foo'],
+                                     incremental=True,
+                                     delete_where={'column': 'lilly',
+                                                   'values': ['a', 'b'],
+                                                   'operator': 'eq'})
+
+# update column
+out.update_column('foo',
+                  ColumnDefinition(data_types=BaseType(dtype=SupportedDataTypes.INTEGER, length='20')))
+
+# add new columns
+out.add_column('note', ColumnDefinition(nullable=False))
+out.add_column('test1')
+out.add_columns(['test2', 'test3', 'test4'])
+
+# add new typed column
+out.add_column('id', ColumnDefinition(primary_key=True,
+                                      data_types={'snowflake': DataType(dtype="INTEGER", length='200')})
+               )
+
+out.add_columns({
+    'phone': ColumnDefinition(primary_key=True,
+                              data_types={'snowflake': DataType(dtype="INTEGER", length='200'),
+                                          'bigquery': DataType(dtype="BIGINT")}),
+    'new2': ColumnDefinition(data_types={'snowflake': DataType(dtype="INTEGER", length='200')}),
+                 })
+
+# delete columns
+out.delete_column('bar')
+out.delete_columns(['test2', 'test3'])
+
+
+# write some content
+with open(out.full_path, 'w') as result:
+    result.write('line')
+    
+# write manifest
+ci.write_manifest(out)
+```
+
 **Example:**
 
 ```python