castor-extractor 0.5.3__py3-none-any.whl → 0.5.6__py3-none-any.whl

castor_extractor/visualization/tableau/extract.py

@@ -27,17 +27,17 @@ def iterate_all_data(
 
     logger.info("Extracting WORKBOOK from Tableau API")
     yield TableauAsset.WORKBOOK, deep_serialize(
-        client.fetch(TableauAsset.WORKBOOK)
+        client.fetch(TableauAsset.WORKBOOK),
     )
 
     logger.info("Extracting PUBLISHED DATASOURCE from Tableau API")
     yield TableauAsset.PUBLISHED_DATASOURCE, deep_serialize(
-        client.fetch(TableauAsset.PUBLISHED_DATASOURCE)
+        client.fetch(TableauAsset.PUBLISHED_DATASOURCE),
     )
 
     logger.info("Extracting PROJECT from Tableau API")
     yield TableauAsset.PROJECT, deep_serialize(
-        client.fetch(TableauAsset.PROJECT)
+        client.fetch(TableauAsset.PROJECT),
     )
 
     logger.info("Extracting USAGE from Tableau API")
@@ -45,22 +45,22 @@ def iterate_all_data(
 
     logger.info("Extracting WORKBOOK_TO_DATASOURCE from Tableau API")
     yield TableauAsset.WORKBOOK_TO_DATASOURCE, deep_serialize(
-        client.fetch(TableauAsset.WORKBOOK_TO_DATASOURCE)
+        client.fetch(TableauAsset.WORKBOOK_TO_DATASOURCE),
     )
 
     logger.info("Extracting DATASOURCE from Tableau API")
     yield TableauAsset.DATASOURCE, deep_serialize(
-        client.fetch(TableauAsset.DATASOURCE)
+        client.fetch(TableauAsset.DATASOURCE),
     )
 
     logger.info("Extracting CUSTOM_SQL_TABLE from Tableau API")
     yield TableauAsset.CUSTOM_SQL_TABLE, deep_serialize(
-        client.fetch(TableauAsset.CUSTOM_SQL_TABLE)
+        client.fetch(TableauAsset.CUSTOM_SQL_TABLE),
     )
 
     logger.info("Extracting CUSTOM_SQL_QUERY from Tableau API")
     yield TableauAsset.CUSTOM_SQL_QUERY, deep_serialize(
-        client.fetch(TableauAsset.CUSTOM_SQL_QUERY)
+        client.fetch(TableauAsset.CUSTOM_SQL_QUERY),
     )
 
     logger.info("Extracting FIELD from Tableau API")

castor_extractor/visualization/tableau/gql_fields.py

@@ -169,19 +169,19 @@ QUERY_FIELDS: Dict[TableauAsset, QueryInfo] = {
         {
             FIELDS: GQLQueryFields.CUSTOM_SQL_TABLE,
             OBJECT_TYPE: TableauGraphqlAsset.CUSTOM_SQL,
-        }
+        },
     ],
     TableauAsset.CUSTOM_SQL_QUERY: [
         {
             FIELDS: GQLQueryFields.CUSTOM_SQL_QUERY,
             OBJECT_TYPE: TableauGraphqlAsset.CUSTOM_SQL,
-        }
+        },
     ],
     TableauAsset.DATASOURCE: [
         {
             FIELDS: GQLQueryFields.DATASOURCE,
             OBJECT_TYPE: TableauGraphqlAsset.DATASOURCE,
-        }
+        },
     ],
     TableauAsset.FIELD: [
         {
@@ -205,6 +205,6 @@ QUERY_FIELDS: Dict[TableauAsset, QueryInfo] = {
         {
             FIELDS: GQLQueryFields.WORKBOOK_TO_DATASOURCE,
             OBJECT_TYPE: TableauGraphqlAsset.WORKBOOK_TO_DATASOURCE,
-        }
+        },
     ],
 }

castor_extractor/visualization/tableau/tests/unit/graphql/paginated_object_test.py

@@ -11,7 +11,8 @@ from ....constants import TABLEAU_SERVER_VERSION
 from ..utils import KEYS
 
 TEST_ASSET_DIR = os.path.join(
-    os.path.dirname(__file__), "../assets/graphql/metadata"
+    os.path.dirname(__file__),
+    "../assets/graphql/metadata",
 )
 
 METADATA_1 = os.path.join(TEST_ASSET_DIR, "metadata_1_get.json")

castor_extractor/visualization/tableau/tests/unit/rest_api/auth_test.py

@@ -27,11 +27,14 @@ class AuthTests(unittest.TestCase):
             self._client.login()
 
         self.assertEqual(
-            "eIX6mvFsqyansa4KqEI1UwOpS8ggRs2l", self._client._server.auth_token
+            "eIX6mvFsqyansa4KqEI1UwOpS8ggRs2l",
+            self._client._server.auth_token,
         )
         self.assertEqual(
-            "6b7179ba-b82b-4f0f-91ed-812074ac5da6", self._client._server.site_id
+            "6b7179ba-b82b-4f0f-91ed-812074ac5da6",
+            self._client._server.site_id,
         )
         self.assertEqual(
-            "1a96d216-e9b8-497b-a82a-0b899a965e01", self._client._server.user_id
+            "1a96d216-e9b8-497b-a82a-0b899a965e01",
+            self._client._server.user_id,
         )

castor_extractor/visualization/tableau/tests/unit/rest_api/credentials_test.py

@@ -7,7 +7,7 @@ from ....client.credentials import CredentialsApi
 
 
 def test_default_site_id():
-    creds = CredentialsApi(
+    creds = CredentialsApi(  # noqa: S106
         server_url="url",
         user="test_user",
         password="test_pwd",

castor_extractor/visualization/tableau/tests/unit/rest_api/usages_test.py

@@ -41,7 +41,8 @@ class UsageTests(unittest.TestCase):
             response_xml = f.read().decode("utf-8")
         with requests_mock.mock() as m:
             m.get(
-                self.baseurl + "?includeUsageStatistics=true", text=response_xml
+                self.baseurl + "?includeUsageStatistics=true",
+                text=response_xml,
             )
             results = self._client._fetch_usages(False)
 

castor_extractor/warehouse/abstract/extract.py

@@ -48,7 +48,6 @@ class ExtractionProcessor:
         return decorated_execute(query)
 
     def _results(self, asset: WarehouseAsset) -> Iterator[dict]:
-
         data: Iterator[dict] = iter([])
         queries = self._query_builder.build(asset)
         total = len(queries)
@@ -65,7 +64,9 @@ class ExtractionProcessor:
         return data
 
     def extract(
-        self, asset: WarehouseAsset, skip_existing: bool = False
+        self,
+        asset: WarehouseAsset,
+        skip_existing: bool = False,
     ) -> str:
         """
         Process extraction for the given asset and returns the location of extracted data

castor_extractor/warehouse/abstract/time_filter_test.py

@@ -4,7 +4,6 @@ from .time_filter import TimeFilter
 
 
 def test__time_filter(freezer):
-
     tf = TimeFilter(day=datetime.date(1992, 5, 13))
 
     assert tf.to_dict() == {

castor_extractor/warehouse/bigquery/client_test.py

@@ -37,7 +37,7 @@ class MockBigQueryClient(BigQueryClient):
                 MockProject("project_1"),
                 MockProject("project_2"),
                 MockProject("hidden_project"),
-            ]
+            ],
         )
         fake_client.list_datasets = lambda project: client_project_structure[
             project

castor_extractor/warehouse/bigquery/extract.py

@@ -30,7 +30,7 @@ BIGQUERY_ASSETS: SupportedAssets = OrderedDict(
         WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
         WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
         WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
-    }
+    },
 )
 
 
@@ -66,7 +66,8 @@ def extract_all(**kwargs) -> None:
     logger.info(f"Available projects: {client.get_projects()}\n")
 
     query_builder = BigQueryQueryBuilder(
-        regions=client.get_regions(), datasets=client.get_datasets()
+        regions=client.get_regions(),
+        datasets=client.get_datasets(),
     )
 
     storage = LocalStorage(directory=output_directory)

castor_extractor/warehouse/bigquery/query.py

@@ -61,7 +61,8 @@ class BigQueryQueryBuilder(AbstractQueryBuilder):
         sync_tags: Optional[bool] = False,
     ):
         super().__init__(
-            time_filter=time_filter, duplicated=BIGQUERY_DUPLICATES
+            time_filter=time_filter,
+            duplicated=BIGQUERY_DUPLICATES,
         )
         self._regions = regions
         self._datasets = datasets
@@ -110,7 +111,7 @@ class BigQueryQueryBuilder(AbstractQueryBuilder):
 
         if asset in REGION_REQUIRED:
             logger.info(
-                f"\tWill run queries with following region params: {self._regions}"
+                f"\tWill run queries with following region params: {self._regions}",
             )
             return [
                 self._format(query, {"project": project, "region": region})
@@ -119,7 +120,7 @@ class BigQueryQueryBuilder(AbstractQueryBuilder):
 
         if asset in DATASET_REQUIRED:
             logger.info(
-                f"\tWill run queries with following dataset params: {self._datasets}"
+                f"\tWill run queries with following dataset params: {self._datasets}",
             )
             return [
                 self._format(query, {"project": project, "dataset": dataset})

castor_extractor/warehouse/postgres/extract.py

@@ -23,12 +23,12 @@ POSTGRES_ASSETS: SupportedAssets = OrderedDict(
             WarehouseAsset.GROUP,
             WarehouseAsset.USER,
         ),
-    }
+    },
 )
 
 
 POSTGRES_USER = "CASTOR_POSTGRES_USER"
-POSTGRES_PASSWORD = "CASTOR_POSTGRES_PASSWORD"
+POSTGRES_PASSWORD = "CASTOR_POSTGRES_PASSWORD"  # noqa: S105
 POSTGRES_HOST = "CASTOR_POSTGRES_HOST"
 POSTGRES_PORT = "CASTOR_POSTGRES_PORT"
 POSTGRES_DATABASE = "CASTOR_POSTGRES_DATABASE"
@@ -59,7 +59,9 @@ def extract_all(**kwargs) -> None:
     storage = LocalStorage(directory=output_directory)
 
     extractor = ExtractionProcessor(
-        client=client, query_builder=query_builder, storage=storage
+        client=client,
+        query_builder=query_builder,
+        storage=storage,
     )
 
     for group in POSTGRES_ASSETS.values():

castor_extractor/warehouse/redshift/client_test.py

@@ -25,7 +25,6 @@ def test_redshift_client_execute(
     mock_engine_options,
     mock_build_uri,
 ):
-
     mock_connect.return_value = Mock(execute=fake_execute)
 
     credentials = {}

castor_extractor/warehouse/redshift/extract.py

@@ -26,11 +26,11 @@ REDSHIFT_ASSETS: SupportedAssets = OrderedDict(
             WarehouseAsset.USER,
             WarehouseAsset.GROUP,
         ),
-    }
+    },
 )
 
 REDSHIFT_USER = "CASTOR_REDSHIFT_USER"
-REDSHIFT_PASSWORD = "CASTOR_REDSHIFT_PASSWORD"
+REDSHIFT_PASSWORD = "CASTOR_REDSHIFT_PASSWORD"  # noqa: S105
 REDSHIFT_HOST = "CASTOR_REDSHIFT_HOST"
 REDSHIFT_PORT = "CASTOR_REDSHIFT_PORT"
 REDSHIFT_DATABASE = "CASTOR_REDSHIFT_DATABASE"
@@ -61,7 +61,9 @@ def extract_all(**kwargs) -> None:
     storage = LocalStorage(directory=output_directory)
 
     extractor = ExtractionProcessor(
-        client=client, query_builder=query_builder, storage=storage
+        client=client,
+        query_builder=query_builder,
+        storage=storage,
     )
 
     for group in REDSHIFT_ASSETS.values():

castor_extractor/warehouse/snowflake/client.py

@@ -104,7 +104,7 @@ class SnowflakeClient(SqlalchemyClient):
         """check and set warehouse"""
         if self._warehouse:
             logger.info(
-                f"Warehouse was provided in arguments: {self._warehouse}"
+                f"Warehouse was provided in arguments: {self._warehouse}",
             )
             _use(connection, UseResource.WAREHOUSE, self._warehouse)
             return

castor_extractor/warehouse/snowflake/client_test.py

@@ -48,7 +48,7 @@ def test_build_uri(_):
 
 @patch.object(SqlalchemyClient, "__init__")
 @patch(
-    "source.packages.extractor.castor_extractor.warehouse.snowflake.client._use"
+    "source.packages.extractor.castor_extractor.warehouse.snowflake.client._use",
 )
 def test_role(mocked_used, _):
     client = get_snowflake_connection()

castor_extractor/warehouse/snowflake/extract.py

@@ -30,12 +30,12 @@ SNOWFLAKE_ASSETS: SupportedAssets = OrderedDict(
             WarehouseAsset.USER,
         ),
         WarehouseAssetGroup.LINEAGE: LINEAGE_ASSETS,
-    }
+    },
 )
 
 SNOWFLAKE_ACCOUNT = "CASTOR_SNOWFLAKE_ACCOUNT"
 SNOWFLAKE_USER = "CASTOR_SNOWFLAKE_USER"
-SNOWFLAKE_PASSWORD = "CASTOR_SNOWFLAKE_PASSWORD"
+SNOWFLAKE_PASSWORD = "CASTOR_SNOWFLAKE_PASSWORD"  # noqa: S105
 
 
 def _credentials(params: dict) -> dict:
@@ -69,7 +69,9 @@ def extract_all(**kwargs) -> None:
     storage = LocalStorage(directory=output_directory)
 
     extractor = ExtractionProcessor(
-        client=client, query_builder=query_builder, storage=storage
+        client=client,
+        query_builder=query_builder,
+        storage=storage,
     )
 
     for group in SNOWFLAKE_ASSETS.values():

castor_extractor/warehouse/synapse/extract.py

@@ -18,5 +18,5 @@ SYNAPSE_ASSETS: SupportedAssets = OrderedDict(
         WarehouseAssetGroup.QUERY: QUERIES_ASSETS,
         WarehouseAssetGroup.VIEW_DDL: VIEWS_ASSETS,
         WarehouseAssetGroup.ROLE: (WarehouseAsset.USER,),
-    }
+    },
 )

{castor_extractor-0.5.3.dist-info → castor_extractor-0.5.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: castor-extractor
-Version: 0.5.3
+Version: 0.5.6
 Summary: Extract your metadata assets.
 Home-page: https://www.castordoc.com/
 License: EULA
@@ -22,7 +22,7 @@ Provides-Extra: redshift
 Provides-Extra: snowflake
 Provides-Extra: tableau
 Requires-Dist: cachetools (>=4.2.4,<5.0.0)
-Requires-Dist: certifi (==2021.10.8)
+Requires-Dist: certifi (==2023.7.22)
 Requires-Dist: charset-normalizer (>=2.0.7,<3.0.0)
 Requires-Dist: click (>=8.0,<8.1)
 Requires-Dist: google-api-core (>=2.1.1,<3.0.0)