tinybird-cli 5.22.3.dev0__py3-none-any.whl → 6.0.2.dev0__py3-none-any.whl

tinybird/connectors.py

@@ -1,422 +0,0 @@
-import logging
-import os
-import time
-from abc import ABC, abstractmethod
-from datetime import datetime
-from typing import Any, Dict, List, Optional, Tuple
-from urllib.parse import unquote
-
-import requests
-
-from .client import ConnectorNothingToLoad
-
-UNINSTALLED_CONNECTORS = []
-
-# common dependencies to snowflake and bigquery connectors
-try:
-    from google.cloud import storage
-    from google.cloud.storage.blob import Blob
-    from google.oauth2 import service_account
-except ImportError:
-    UNINSTALLED_CONNECTORS += ["bigquery", "snowflake"]
-
-try:
-    from google.cloud import bigquery
-except ImportError:
-    UNINSTALLED_CONNECTORS += ["bigquery"]
-
-try:
-    import googleapiclient.discovery
-    import snowflake.connector
-    from google.api_core.exceptions import PreconditionFailed
-except ImportError:
-    UNINSTALLED_CONNECTORS += ["snowflake"]
-
-UNINSTALLED_CONNECTORS = list(set(UNINSTALLED_CONNECTORS))
-
-logger = logging.getLogger("tinybird-connect")
-
-
-def _now():
-    return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
-
-def _log(text):
-    logger.info(f"{_now()} - {text}")
-
-
-class GCS:
-    MAX_FILE_SIZE = int(5 * 1024**3)
-    MAX_COMPOSE = 31
-    READ_TIMEOUT = 3
-
-    def __init__(self, options):
-        self.options = options
-        filename = options["service_account"]
-        if not filename:
-            filename = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
-        self.credentials = service_account.Credentials.from_service_account_file(
-            filename=filename,
-            scopes=["https://www.googleapis.com/auth/cloud-platform"],
-        )
-        self.storage_client = storage.Client(project=options["project_id"], credentials=self.credentials)
-
-    def gcs_url(self):
-        return f"gcs://{self.bucket_name()}/"
-
-    def gs_url(self):
-        return f"gs://{self.bucket_name()}/"
-
-    def bucket_name(self):
-        return self.options["bucket_name"]
-
-    def sign(self, blob_name):
-        bucket = self.storage_client.get_bucket(self.bucket_name())
-        blob = bucket.get_blob(blob_name)
-        if not blob:
-            raise Exception("Warning: File not found. This probably just means there is no new data to load")
-        return blob.generate_signed_url(expiration=86400, version="v4", scheme="https")
-
-    def grant_access(self, member_name):
-        role_name = self.get_role_name()
-        service = googleapiclient.discovery.build("iam", "v1", credentials=self.credentials)
-        result = service.roles().list(parent="projects/" + self.options["project_id"]).execute()
-        exists = False if "roles" not in result else any(role["title"] == role_name for role in result["roles"])
-        if not exists:
-            role_name = self.create_custom_role(service)
-        self.add_member_to_bucket(member_name, role_name, self.bucket_name())
-
-    def get_role_name(self):
-        return f"{self.bucket_name().replace('-', '')}_5".lower()
-
-    def create_custom_role(self, service):
-        service.projects().roles().create(
-            parent=f"projects/{self.options['project_id']}",
-            body={
-                "roleId": self.get_role_name(),
-                "role": {
-                    "title": self.get_role_name(),
-                    "description": self.get_role_name(),
-                    "includedPermissions": [
-                        "storage.objects.create",
-                        "storage.buckets.get",
-                        "storage.objects.delete",
-                        "storage.objects.get",
-                        "storage.objects.list",
-                    ],
-                    "stage": "GA",
-                },
-            },
-        ).execute()
-        return self.get_role_name()
-
-    def add_member_to_bucket(self, member_name, iam_role, bucket_name, retry=True):
-        try:
-            project_id = self.options["project_id"]
-            role = f"projects/{project_id}/roles/{iam_role}"
-            member = f"serviceAccount:{member_name}"
-
-            bucket = self.storage_client.bucket(bucket_name)
-            policy = bucket.get_iam_policy(requested_policy_version=3)
-            policy.bindings.append({"role": role, "members": {member}})
-            bucket.set_iam_policy(policy)
-        except PreconditionFailed as e:
-            _log(str(e))
-            if retry:
-                _log("retrying...")
-                self.add_member_to_bucket(member_name, iam_role, bucket_name, retry=False)
-
-    def compose(self, source_prefix: str, destination_blob_name: str, mode: str) -> List[str]:
-        bucket = self.storage_client.bucket(self.bucket_name())
-        retries: int = 0
-        sources: Optional[List[Blob]] = None
-        # for some reason when this list_blobs run in GH actions takes forever and fails
-        # so retry many times with a lower timeout
-        while not sources and retries < 5:
-            try:
-                sources = list(
-                    self.storage_client.list_blobs(
-                        self.bucket_name(), prefix=source_prefix, timeout=self.READ_TIMEOUT, fields="items(name,size)"
-                    )
-                )
-            except requests.exceptions.ReadTimeout:
-                _log("exception fetching blob list from GCS")
-            retries += 1
-
-        if sources is None:
-            raise Exception("couldn't load list of blobs from GCS")
-
-        _log(f"Exported {len(sources)} CSV files")
-        if len(sources) == 0:
-            raise ConnectorNothingToLoad("Warning: Nothing to load. Aborting next steps")
-
-        def compose_with_limits(
-            sources: List[Blob], max_file_size: float = float("inf"), iteration: int = 0
-        ) -> Tuple[List[Blob], List[Blob], int]:
-            blobs: List[Blob] = []
-            blobs_to_delete: List[Blob] = []
-            chunk: List[Blob] = []
-            chunk_size: int = 0
-            count: int = 0
-            for i, blob in enumerate(sources):
-                chunk.append(blob)
-                chunk_size += blob.size
-
-                if chunk_size >= max_file_size or len(chunk) >= self.MAX_COMPOSE or i == len(sources) - 1:
-                    destination_name: str = f"{destination_blob_name}_{iteration}_part{count}.csv"
-                    destination: Blob = bucket.blob(destination_name)
-                    destination.content_type = "text/csv"
-                    destination.compose(chunk)
-
-                    blobs.append(destination)
-                    blobs_to_delete += chunk
-                    chunk = []
-                    chunk_size = 0
-                    count += 1
-
-            return blobs, blobs_to_delete, iteration + 1
-
-        blobs: List[Blob] = []
-        blobs_to_delete: List[Blob] = []
-        if mode == "append":
-            blobs, blobs_to_delete, _ = compose_with_limits(sources, max_file_size=self.MAX_FILE_SIZE)
-        elif mode == "replace":
-            blobs = sources
-            iteration: int = 0
-            while len(blobs) > 1:
-                blobs, blobs_to_delete_iteration, iteration = compose_with_limits(blobs, iteration=iteration)
-                blobs_to_delete += blobs_to_delete_iteration
-
-        try:
-            _log("Removing temp blobs")
-            for blob in blobs_to_delete:
-                blob.delete()
-        except Exception:
-            pass
-
-        _log("Done composing!")
-        return [blob.name for blob in blobs]
-
-    def delete(self, blob_name):
-        bucket = self.storage_client.bucket(self.bucket_name())
-        bucket.delete_blob(blob_name)
-
-
-class Connector(ABC):
-    def __init__(self, options: Dict[str, Any]):
-        self.options = options
-        self.options["gcs"] = {
-            "project_id": options["project_id"],
-            "service_account": options["service_account"],
-            "sign_service_account": options["service_account"],
-            "bucket_name": options["bucket_name"],
-        }
-        self.gcs = GCS(self.options["gcs"])
-
-    @abstractmethod
-    def export_to_gcs(self, sql: str, destination: str, mode: str) -> List[str]:
-        pass
-
-    @abstractmethod
-    def clean(self, blob_name):
-        pass
-
-    @abstractmethod
-    def datasource_analyze(self, resource):
-        pass
-
-
-class BigQuery(Connector):
-    def __init__(self, options):
-        super().__init__(options)
-        self.configure()
-
-    def configure(self):
-        pass
-
-    def datasource_analyze(self, resource):
-        raise Exception("BigQuery does not support data source analyze")
-
-    def export_to_gcs(self, sql: str, destination: str, mode: str) -> List[str]:
-        mm = str(int(round(time.time() * 1000)))
-        destination = f"{destination}_{mm}"
-        with_headers: bool = self.options.get("with_headers", False)
-        sql = f"""
-                EXPORT DATA OPTIONS(
-                    uri='{self.gcs.gs_url()}{destination}*.csv',
-                    format='CSV',
-                    overwrite=true,
-                    header={"true" if with_headers else "false"},
-                    field_delimiter=',') AS
-                    {sql}
-                """
-        self.execute(sql)
-        urls: List[str] = self.gcs.compose(destination, f"{destination}_final", mode)
-        return [unquote(self.gcs.sign(url)) for url in urls]
-
-    def execute(self, sql, result=False):
-        client = self.connector()
-        job_config = bigquery.QueryJobConfig()
-
-        query_job = client.query(
-            sql,
-            job_config=job_config,
-        )
-
-        return query_job.result()  # Waits for the query to finish
-
-    def connector(self):
-        credentials = service_account.Credentials.from_service_account_file(
-            filename=self.options["service_account"],
-            scopes=["https://www.googleapis.com/auth/cloud-platform"],
-        )
-        self.credentials = credentials
-
-        return bigquery.Client(credentials=credentials, project=credentials.project_id)
-
-    def clean(self, blob_name):
-        self.gcs.delete(blob_name)
-
-
-class Snowflake(Connector):
-    DEFAULT_STORAGE_INTEGRATION = "tb__gcs_int"
-    DEFAULT_STAGE = "tb__gcs_stage"
-
-    def __init__(self, options):
-        super().__init__(options)
-        self.configure()
-
-    def configure(self):
-        self.create_storage_integration()
-        self.create_stage()
-        self.gcs.grant_access(self.get_sf_member_name())
-
-    def create_storage_integration(self):
-        sql = f"""
-                    create storage integration {self.storage_integration()}
-                        type = external_stage
-                        storage_provider = gcs
-                        enabled = true
-                        storage_allowed_locations = ('{self.gcs.gcs_url()}');
-                """
-        try:
-            self.execute(sql)
-        except Exception as e:
-            if "already exists" in str(e):
-                pass
-            else:
-                raise e
-
-    def get_sf_member_name(self):
-        try:
-            sql = f"DESC STORAGE INTEGRATION {self.storage_integration()};"
-            result = self.execute(sql, result=True)
-            for row in result:
-                if row["property"] == "STORAGE_GCP_SERVICE_ACCOUNT":
-                    return row["property_value"]
-        except Exception:
-            return None
-
-    def create_stage(self):
-        sql = f"""
-                    create stage "{self.options["schema"]}".{self.stage()}
-                        url='{self.gcs.gcs_url()}'
-                        storage_integration = {self.storage_integration()};
-                """
-        try:
-            self.execute(sql)
-        except Exception as e:
-            if "already exists" in str(e):
-                pass
-
-    def export_to_gcs(self, sql: str, destination: str, mode: str) -> List[str]:
-        mm = str(int(round(time.time() * 1000)))
-        destination = f"{destination}_{mm}"
-        with_headers: bool = self.options.get("with_headers", False)
-        sql = f"""copy into '@{self.stage()}/{destination}'
-                    from ({sql})
-                    overwrite = true
-                    file_format = (TYPE=CSV COMPRESSION=NONE ESCAPE_UNENCLOSED_FIELD=NONE FIELD_DELIMITER='|' FIELD_OPTIONALLY_ENCLOSED_BY='"' null_if=())
-                    header = {"true" if with_headers else "false"}
-                    max_file_size = 2500000000;
-                """
-        self.execute(sql)
-        urls: List[str] = self.gcs.compose(destination, f"{destination}_final", mode)
-        return [unquote(self.gcs.sign(url)) for url in urls]
-
-    def execute(self, sql, result=False):
-        ctx = self.connector()
-        cs = ctx.cursor(snowflake.connector.DictCursor)
-        try:
-            cs.execute(f"use role {self.options['role']};")
-            cs.execute(f"use warehouse {self.options['warehouse']};")
-            cs.execute(sql)
-            if result:
-                result = cs.fetchall()
-        finally:
-            cs.close()
-        ctx.close()
-        return result
-
-    def datasource_analyze(self, resource):
-        """returns .datasource file for resource"""
-        # TODO: deal with the right precisions for Integers
-        YES = "Y"
-        NO = "N"
-
-        def from_snowflake_type(t: str, nullable=NO) -> str:
-            """transforms snowflake types to CH ones"""
-            the_type = "String"
-            if t.startswith("NUMBER"):
-                the_type = "Int32"
-            if t.startswith(("FLOAT", "DOUBLE", "REAL", "NUMERIC", "DECIMAL")):
-                the_type = "Float32"
-            if t == "DATE":
-                the_type = "Date"
-            if t == "DATETIME" or t.startswith("TIMESTAMP"):
-                the_type = "DateTime"
-            if nullable == YES:
-                the_type = f"Nullable({the_type})"
-            return the_type
-
-        result = self.execute(f"DESCRIBE TABLE {resource};", result=True)
-        sql = []
-        columns = []
-        for row in result:
-            if row["kind"] == "COLUMN":
-                sql.append(f"{row['name']} {from_snowflake_type(row['type'], row['null?'])}")
-                columns.append(
-                    {
-                        "path": row["name"],
-                        "name": row["name"],
-                        "present_pct": 1 if row["null?"] != YES else None,
-                        "recommended_type": from_snowflake_type(row["type"]),
-                    }
-                )
-        return {"analysis": {"columns": columns, "schema": ", ".join(sql)}}
-
-    def connector(self):
-        auth = {
-            key: self.options[key]
-            for key in self.options.keys() & {"user", "password", "account", "warehouse", "database", "schema"}
-        }
-        return snowflake.connector.connect(**auth)
-
-    def stage(self):
-        return self.options.get("stage") or self.DEFAULT_STAGE
-
-    def storage_integration(self):
-        return self.options.get("storage_integration") or self.DEFAULT_STORAGE_INTEGRATION
-
-    def clean(self, blob_name):
-        self.gcs.delete(blob_name)
-
-
-connectors = {
-    "snowflake": Snowflake,
-    "bigquery": BigQuery,
-}
-
-
-def create_connector(source: str, params: Dict[str, Any]) -> Connector:
-    return connectors[source](params)

tinybird_cli-5.22.3.dev0.dist-info/RECORD

@@ -1,45 +0,0 @@
-tinybird/__cli__.py,sha256=bnGPJ7820E46g4vYb-IUKKWXtNxDC9dv3dxTnquqY4A,237
-tinybird/check_pypi.py,sha256=_4NkharLyR_ELrAdit-ftqIWvOf7jZNPt3i76frlo9g,975
-tinybird/client.py,sha256=VYKuR10NwYmvyvC5QsJPE0H4k9dLDb_TxwxYbEyP1Xs,55592
-tinybird/config.py,sha256=4URIMvEO-ysbXrd-hGbXDAeymZfdiPEGauJPaUeccXI,7493
-tinybird/connectors.py,sha256=TSux0opkG3gbzTTyRIUfNk4RJoLvTIyp42rdsmmIAYg,15083
-tinybird/context.py,sha256=o4yvlXPkMLmdh-XJl3wpmqPAMeRRz5ScKzKlHHKn_I8,1201
-tinybird/datafile_common.py,sha256=xYnTdUv_yx17QO37S4SlWOK5wHD103LoJe3l8SvpWNk,233167
-tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
-tinybird/feedback_manager.py,sha256=T370HP53XP4EJ-FCxmsuW6v-eG_xyonGr69a_BRVSVw,69978
-tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
-tinybird/sql.py,sha256=7pkwQMUVy0CH5zFgLMZyCx1BeaJSQYC05EmOb9vO3Ls,48694
-tinybird/sql_template.py,sha256=VLbog44F0VY5l5yV3nErz75l4hKko5QxywXi7KugaGM,103297
-tinybird/sql_template_fmt.py,sha256=Ma4qcs-2r8ZXQC4GUmrCqYz34DsnGF8k5lE2Jwnr314,10638
-tinybird/sql_toolset.py,sha256=J4LnRyv-BcoR6Axv24yvPJewCeg0zOZgz-F6ml3q12g,27228
-tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
-tinybird/tb_cli.py,sha256=q1LGAsBVVMJsjR2HK62Pu6vpVtLzNmH8wHrEVUUdVkU,744
-tinybird/tornado_template.py,sha256=p3V4ERIWPRdtQuqkaL1NVjivaFoVXZgPJUxwVPce4nc,41850
-tinybird/ch_utils/constants.py,sha256=yTNizMzgYNBzUc2EV3moBfdrDIggOe9hiuAgWF7sv2c,4333
-tinybird/ch_utils/engine.py,sha256=UEis9pIYg9a7dpGHVYZ2D3pS4yP1MCFfmvzW7Fkrl1U,41270
-tinybird/tb_cli_modules/auth.py,sha256=urjPOQtukbQ2ooRUJmUMXS98mLfpOl64Q9NwG0DQjnw,9022
-tinybird/tb_cli_modules/branch.py,sha256=92jKpb28yZMav4w5s6HboGty8ivtgSIFh1zxvLK3mBo,39348
-tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
-tinybird/tb_cli_modules/cli.py,sha256=E8648ygwOMFaj1d02qYCWtnpQfei1sYBdE2r4DWg4EM,63394
-tinybird/tb_cli_modules/common.py,sha256=VQQjGmbbKR09TWn1lxd_ip4uykqaBcY6UoXB2MlWvgQ,82985
-tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4ewgA,11546
-tinybird/tb_cli_modules/connection.py,sha256=u_H08nnAUvO0GevAT4WoHrTdtIt8BMPvRJMa-vIJaDg,29399
-tinybird/tb_cli_modules/datasource.py,sha256=WM4I1XFkMOBDFglo1GSVIOVELvjjDfs3SC_9zEQUERM,35768
-tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
-tinybird/tb_cli_modules/fmt.py,sha256=edQap4tAqWMWogSIx5zriT75naLi73XTB3NwatmcrFw,3518
-tinybird/tb_cli_modules/job.py,sha256=AG69LPb9MbobA1awwJFZJvxqarDKfRlsBjw2V1zvYqc,2964
-tinybird/tb_cli_modules/pipe.py,sha256=LEPZ6EqQsW525W1UxuMMAGMMtNMJqxz5G3ZsCHO5XTc,31442
-tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
-tinybird/tb_cli_modules/tag.py,sha256=9YHnruPnUNp1IJUe4qcSEMg9EbquIRo--Nxcsbvkvq8,3488
-tinybird/tb_cli_modules/telemetry.py,sha256=W098H6jmS4kpE7hN3tadaREBTf7oMocel-lkKWN0pU8,10466
-tinybird/tb_cli_modules/test.py,sha256=Vf8oK96V81HdKGsT79y6MUz6oz_VrYIwTbRnzzJs4rQ,4350
-tinybird/tb_cli_modules/token.py,sha256=JXATKTlbXohP9ZDZjlz8E4VYG6zrknKZhuz_wh1zBBc,13793
-tinybird/tb_cli_modules/workspace.py,sha256=yg3yb7_GniGJnOy2HqwEzePl47gQD-hywuQJyclZHi0,12455
-tinybird/tb_cli_modules/workspace_members.py,sha256=ksXsjd233y9-sNlz4Qb-meZbX4zn1B84e_bSm2i8rhg,8731
-tinybird/tb_cli_modules/tinyunit/tinyunit.py,sha256=4QlVoHE-O3akOBuIIJm3Mx2ewiZmPTT9vdgqIi9byTw,11267
-tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py,sha256=NHoXcCHPDcKWYLzgP3NViho3Ey-6RV-ynPDzySPrTPE,1817
-tinybird_cli-5.22.3.dev0.dist-info/METADATA,sha256=dRLdOfnK115mYqNO88wOCweM89sO9AjH9AydMho3ECY,81473
-tinybird_cli-5.22.3.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
-tinybird_cli-5.22.3.dev0.dist-info/entry_points.txt,sha256=PKPKuPmA4IfJYnCFHHUiw-aAWZuBomFvwCklv1OyCjE,43
-tinybird_cli-5.22.3.dev0.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird_cli-5.22.3.dev0.dist-info/RECORD,,