visivo 1.0.20__tar.gz

visivo-1.0.20/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.1
+Name: visivo
+Version: 1.0.20
+Summary: 
+Author: Visivo People
+Requires-Python: >=3.10,<3.13
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: Flask (>=2.2.3,<3.0.0)
+Requires-Dist: Jinja2 (>=3.1.2,<4.0.0)
+Requires-Dist: PyYAML (>=6.0,<7.0)
+Requires-Dist: assertpy (>=1.1,<2.0)
+Requires-Dist: click (>=8.1.3,<9.0.0)
+Requires-Dist: deepmerge (>=1.1.0,<2.0.0)
+Requires-Dist: gitpython (>=3.1.32,<4.0.0)
+Requires-Dist: halo (>=0.0.31,<0.0.32)
+Requires-Dist: livereload (>=2.6.3,<3.0.0)
+Requires-Dist: matplotlib (>=3.7.1,<4.0.0)
+Requires-Dist: mkdocs-autolinks-plugin (>=0.7.1,<0.8.0)
+Requires-Dist: networkx (>=3.1,<4.0)
+Requires-Dist: numpy (==1.26.0)
+Requires-Dist: pandas (>=1.5.3,<2.0.0)
+Requires-Dist: psycopg2-binary (>=2.9.5,<3.0.0)
+Requires-Dist: pydantic (>=2.5.0)
+Requires-Dist: pymysql (>=1.1.0,<2.0.0)
+Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
+Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
+Requires-Dist: python-dotenv (>=1.0.0,<2.0.0)
+Requires-Dist: requests (>=2.28.2,<3.0.0)
+Requires-Dist: snowflake-connector-python (>=3.7.0,<4.0.0)
+Requires-Dist: sql-formatter (>=0.6.2,<0.7.0)
+Requires-Dist: sqlalchemy (>=2.0.8,<3.0.0)
+Requires-Dist: termcolor (>=2.4.0,<3.0.0)
+Requires-Dist: tornado (<6.3)
+Project-URL: Documentation, https://docs.visivo.io/

visivo-1.0.20/pyproject.toml

@@ -0,0 +1,57 @@
+[tool.poetry]
+name = "visivo"
+version = "1.0.20"
+description = ""
+authors = ["Visivo People"]
+documentation = "https://docs.visivo.io/"
+
+[tool.poetry.scripts]
+visivo = "visivo.command_line:safe_visivo"
+
+[tool.poetry.dependencies]
+python = " >=3.10,<3.13"
+pydantic = ">=2.5.0"
+requests = "^2.28.2"
+Jinja2 = "^3.1.2"
+PyYAML = "^6.0"
+sql-formatter = "^0.6.2"
+click = "^8.1.3"
+pandas = "^1.5.3"
+Flask = "^2.2.3"
+livereload = "^2.6.3"
+tornado = "< 6.3"
+numpy = "=1.26.0"
+psycopg2-binary = "^2.9.5"
+sqlalchemy = "^2.0.8"
+snowflake-connector-python = "^3.7.0"
+python-dotenv = "^1.0.0"
+networkx = "^3.1"
+matplotlib = "^3.7.1"
+deepmerge = "^1.1.0"
+gitpython = "^3.1.32"
+halo = "^0.0.31"
+mkdocs-autolinks-plugin = "^0.7.1"
+termcolor = "^2.4.0"
+python-dateutil = "^2.9.0.post0"
+pytest-mock = "^3.14.0"
+pymysql = "^1.1.0"
+assertpy = "^1.1"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.2.1"
+factory-boy = "^3.2.1"
+mkdocs = "^1.4.2"
+mkdocs-click = "^0.8.0"
+mkdocs-encryptcontent-plugin = "^2.4.5"
+mkdocs-material = "^9.1.3"
+mkdocs-spellcheck = {extras = ["all"], version = "^1.0.0"}
+symspellpy = "^6.7.7"
+codespell = "^2.2.4"
+mkdocs-macros-plugin = "^0.7.0"
+setuptools = "^67.7.2"
+pytest-env = "^1.0.1"
+requests-mock = "^1.11.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

visivo-1.0.20/visivo/command_line.py

@@ -0,0 +1,63 @@
+import click
+import os
+import importlib
+from dotenv import load_dotenv
+from pydantic import ValidationError
+from visivo.logging.logger import Logger, TypeEnum
+from visivo.parsers.line_validation_error import LineValidationError
+
+from .commands.deploy import deploy
+from .commands.serve import serve
+from .commands.run import run
+from .commands.dist import dist
+from .commands.compile import compile
+from .commands.init import init
+from .commands.test import test
+from .commands.aggregate import aggregate
+from .commands.archive import archive
+
+
+@click.group()
+@click.option("-e", "--env-file", default=".env")
+@click.version_option(version=importlib.metadata.version("visivo"))
+def visivo(env_file):
+    Logger.instance().set_type(TypeEnum.spinner)
+    load_env(env_file)
+
+
+visivo.add_command(init)
+visivo.add_command(compile)
+visivo.add_command(run)
+visivo.add_command(serve)
+visivo.add_command(deploy)
+visivo.add_command(dist)
+visivo.add_command(test)
+visivo.add_command(aggregate)
+visivo.add_command(archive)
+
+
+def load_env(env_file):
+    if os.path.isfile(env_file):
+        Logger.instance().debug(f"Loading env file: {env_file}")
+        load_dotenv(env_file)
+
+
+def safe_visivo():
+    try:
+        visivo(standalone_mode=False)
+    except (ValidationError, LineValidationError) as e:
+        Logger.instance().error(str(e))
+        exit(1)
+    except Exception as e:
+        if "STACKTRACE" in os.environ and os.environ["STACKTRACE"] == "true":
+            raise e
+        Logger.instance().error("An unexpected error has occurred")
+        Logger.instance().error(str(e))
+        Logger.instance().error(
+            "To print more error information add the 'STACKTRACE=true' environment variable."
+        )
+        exit(1)
+
+
+if __name__ == "__main__":
+    safe_visivo()

visivo-1.0.20/visivo/commands/aggregate.py

@@ -0,0 +1,19 @@
+import click
+from visivo.query.aggregator import Aggregator
+from .options import output_dir
+
+
+@click.command()
+@output_dir
+@click.option(
+    "-j", "--json-file", help="The file with the raw json results from the query"
+)
+def aggregate(output_dir, json_file):
+    from visivo.logging.logger import Logger
+
+    Logger.instance().debug("Aggregating")
+
+    from visivo.query.runner import Runner
+
+    Aggregator.aggregate(trace_dir=output_dir, json_file=json_file)
+    Logger.instance().success("Done")

visivo-1.0.20/visivo/commands/archive.py

@@ -0,0 +1,24 @@
+import click
+from .options import user_dir, stage, host
+
+
+@click.command()
+@stage
+@host
+@user_dir
+def archive(stage, host, user_dir):
+    """
+    Archives a stage.  You must specify a stage when deploying a project.
+    """
+    from visivo.logging.logger import Logger
+
+    Logger.instance().debug("Archiving")
+
+    from visivo.commands.archive_phase import archive_phase
+
+    archive_phase(
+        user_dir=user_dir,
+        stage=stage,
+        host=host,
+    )
+    Logger.instance().success("Done")

visivo-1.0.20/visivo/commands/archive_phase.py

@@ -0,0 +1,36 @@
+from visivo.logging.logger import Logger
+import click
+import requests
+import json
+from visivo.commands.utils import get_profile_file, get_profile_token
+
+
+def archive_phase(stage, host, user_dir):
+    profile_token = get_profile_token(get_profile_file(home_directory=user_dir))
+
+    json_headers = {
+        "content-type": "application/json",
+        "Authorization": f"Api-Key {profile_token}",
+    }
+
+    url = f"{host}/api/stages/?name={stage}"
+    response = requests.get(url, headers=json_headers)
+    if response.status_code == 401:
+        raise click.ClickException(f"Token not authorized for host: {host}")
+    if response.status_code == 404:
+        raise click.ClickException(f"404 error raised. Does your user have an account?")
+
+    if len(response.json()) == 0:
+        Logger.instance().debug(f"No stages with name {stage} found")
+        return
+
+    id = response.json()[0]["id"]
+    body = {
+        "archived": True,
+    }
+    url = f"{host}/api/stages/{id}/"
+    response = requests.put(url, data=json.dumps(body), headers=json_headers)
+    if response.status_code == 200:
+        Logger.instance().debug("Stage archived")
+    else:
+        raise click.ClickException(f"There was an unexpected error: {response.content}")

visivo-1.0.20/visivo/commands/compile.py

@@ -0,0 +1,21 @@
+import click
+from .options import output_dir, working_dir, target, name_filter
+
+
+@click.command()
+@target
+@working_dir
+@output_dir
+@name_filter
+def compile(working_dir, output_dir, target, name_filter):
+    """
+    Parses the files in your working directory, extracting visivo configurations and then using those configurations to build the trace queries and a project.json file in your target directory. Queries are not run on compile, just written.
+    """
+    from visivo.logging.logger import Logger
+
+    Logger.instance().debug("Compiling")
+
+    from visivo.commands.compile_phase import compile_phase
+
+    compile_phase(default_target=target, working_dir=working_dir, output_dir=output_dir, name_filter=name_filter)
+    Logger.instance().success("Done")

visivo-1.0.20/visivo/commands/compile_phase.py

@@ -0,0 +1,67 @@
+import click
+import os
+import yaml
+from visivo.discovery.discover import Discover
+from visivo.models.defaults import Defaults
+from visivo.models.models.csv_script_model import CsvScriptModel
+from visivo.models.targets.target import Target
+from visivo.models.models.model import Model
+from visivo.models.base.parent_model import ParentModel
+from visivo.parsers.parser_factory import ParserFactory
+from visivo.parsers.serializer import Serializer
+from visivo.query.query_string_factory import QueryStringFactory
+from visivo.query.trace_tokenizer import TraceTokenizer
+from visivo.query.query_writer import QueryWriter
+from visivo.logging.logger import Logger
+
+
+def compile_phase(
+    default_target: str, working_dir: str, output_dir: str, name_filter: str = None
+):
+    Logger.instance().debug("Compiling project")
+    discover = Discover(working_directory=working_dir)
+    parser = ParserFactory().build(
+        project_file=discover.project_file, files=discover.files
+    )
+    project = None
+    try:
+        project = parser.parse()
+        if not project.defaults:
+            project.defaults = Defaults()
+        if default_target:
+            project.defaults.target_name = default_target
+    except yaml.YAMLError as e:
+        message = "\n"
+        if hasattr(e, "problem_mark"):
+            mark = e.problem_mark
+            message = f"\n Error position: line:{mark.line+1} column:{mark.column+1}\n"
+        raise click.ClickException(
+            f"There was an error parsing the yml file(s):{message} {e}"
+        )
+
+    os.makedirs(output_dir, exist_ok=True)
+    with open(f"{output_dir}/project.json", "w") as fp:
+        serializer = Serializer(project=project)
+        fp.write(serializer.dereference().model_dump_json(exclude_none=True))
+
+    dag = project.dag()
+    for trace in project.filter_traces(name_filter=name_filter):
+        model = ParentModel.all_descendants_of_type(
+            type=Model, dag=dag, from_node=trace
+        )[0]
+        if isinstance(model, CsvScriptModel):
+            target = model.get_sqlite_target(output_dir=output_dir)
+        else:
+            target = ParentModel.all_descendants_of_type(
+                type=Target, dag=dag, from_node=model
+            )[0]
+        tokenized_trace = TraceTokenizer(
+            trace=trace, model=model, target=target
+        ).tokenize()
+        query_string = QueryStringFactory(tokenized_trace=tokenized_trace).build()
+        QueryWriter(
+            trace=trace, query_string=query_string, output_dir=output_dir
+        ).write()
+
+    Logger.instance().debug("Project compiled")
+    return project

visivo-1.0.20/visivo/commands/deploy.py

@@ -0,0 +1,28 @@
+import click
+from visivo.commands.options import output_dir, working_dir, user_dir, stage, host
+
+
+@click.command()
+@working_dir
+@output_dir
+@stage
+@host
+@user_dir
+def deploy(working_dir, user_dir, output_dir, stage, host):
+    """
+    Sends the current version of your project, traces & data to app.visivo.io where it can be viewed by other users on your account. You must specify a stage when deploying a project. The stage allows multiple versions of your project to exist remotely. This is very useful for setting up different dev, CI and production environments.
+    """
+    from visivo.logging.logger import Logger
+
+    Logger.instance().debug("Deploying")
+
+    from visivo.commands.deploy_phase import deploy_phase
+
+    url = deploy_phase(
+        user_dir=user_dir,
+        working_dir=working_dir,
+        output_dir=output_dir,
+        stage=stage,
+        host=host,
+    )
+    Logger.instance().success(f"Deployed to: '{host}{url}'")

visivo-1.0.20/visivo/commands/deploy_phase.py

@@ -0,0 +1,75 @@
+import click
+import requests
+import json
+from visivo.commands.utils import get_profile_file, get_profile_token
+from visivo.discovery.discover import Discover
+from visivo.logging.logger import Logger
+from visivo.parsers.serializer import Serializer
+from visivo.parsers.parser_factory import ParserFactory
+
+
+def deploy_phase(working_dir, user_dir, output_dir, stage, host):
+    """
+    Sends the current version of your project, traces & data to app.visivo.io where it can be viewed by other users on your account. You must specify a stage when deploying a project. The stage allows multiple versions of your project to exist remotely. This is very useful for setting up different dev, CI and production environments.
+    """
+    profile_token = get_profile_token(get_profile_file(home_directory=user_dir))
+
+    discover = Discover(working_directory=working_dir, home_directory=user_dir)
+    parser = ParserFactory().build(
+        project_file=discover.project_file, files=discover.files
+    )
+    project = parser.parse()
+    serializer = Serializer(project=project)
+    project_json = json.loads(
+        serializer.dereference().model_dump_json(exclude_none=True)
+    )
+
+    body = {
+        "project_json": project_json,
+        "name": project_json["name"],
+        "cli_version": project_json["cli_version"],
+        "stage": stage,
+    }
+    json_headers = {
+        "content-type": "application/json",
+        "Authorization": f"Api-Key {profile_token}",
+    }
+    form_headers = {
+        "Authorization": f"Api-Key {profile_token}",
+    }
+
+    url = f"{host}/api/projects/"
+    response = requests.post(url, data=json.dumps(body), headers=json_headers)
+    if response.status_code == 401:
+        raise click.ClickException(f"Token not authorized for host: {host}")
+    if response.status_code == 404:
+        raise click.ClickException(f"404 error raised. Does your user have an account?")
+    if response.status_code == 201:
+        Logger.instance().debug("Project uploaded")
+        project_data = response.json()
+        project_id = project_data["id"]
+        project_url = project_data["url"]
+
+        for trace in project.trace_objs:
+            url = f"{host}/api/files/"
+
+            data_file = f"{output_dir}/{trace.name}/data.json"
+            files = {"file": open(data_file, "rb")}
+            response = requests.post(url, files=files, data={}, headers=form_headers)
+            if response.status_code != 201:
+                raise click.ClickException(f"Trace '{trace.name}' data not uploaded")
+            Logger.instance().debug(f"Trace '{trace.name}' data uploaded")
+            url = f"{host}/api/traces/"
+            body = {
+                "name": trace.name,
+                "project_id": project_id,
+                "data_file_id": response.json()["id"],
+            }
+            response = requests.post(url, data=json.dumps(body), headers=json_headers)
+            if response.status_code != 201:
+                Logger.instance().debug(response.json())
+                raise click.ClickException(f"Trace '{trace.name}' not created")
+            Logger.instance().debug(f"Trace '{trace.name}' created")
+        return project_url
+    else:
+        raise click.ClickException(f"There was an unexpected error: {response.content}")

visivo-1.0.20/visivo/commands/dist.py

@@ -0,0 +1,38 @@
+import click
+from visivo.commands.options import (
+    dist_dir,
+    name_filter,
+    output_dir,
+    target,
+    threads,
+    working_dir,
+)
+
+
+@click.command()
+@target
+@working_dir
+@output_dir
+@dist_dir
+@name_filter
+@threads
+def dist(output_dir, working_dir, dist_dir, target, name_filter, threads):
+    """
+    Creates a distributable version of this dashboard and stores it in a 'dist' folder.
+    This folder can be statically deployed to any web server, hosting service, or bucket storage.
+    """
+    from visivo.logging.logger import Logger
+
+    Logger.instance().debug("Creating Dist")
+
+    from visivo.commands.dist_phase import dist_phase
+
+    dist_phase(
+        output_dir=output_dir,
+        working_dir=working_dir,
+        dist_dir=dist_dir,
+        default_target=target,
+        name_filter=name_filter,
+        threads=threads,
+    )
+    Logger.instance().success(f"Created dist folder")

visivo-1.0.20/visivo/commands/dist_phase.py

@@ -0,0 +1,37 @@
+import pkg_resources
+
+VIEWER_PATH = pkg_resources.resource_filename("visivo", "viewer/")
+
+
+def dist_phase(output_dir, working_dir, dist_dir, default_target, name_filter, threads):
+    from visivo.commands.run_phase import run_phase
+    import os
+    import json
+    import shutil
+    from glob import glob
+
+    run_phase(
+        output_dir=output_dir,
+        working_dir=working_dir,
+        default_target=default_target,
+        name_filter=name_filter,
+        threads=threads,
+    )
+
+    os.makedirs(f"{dist_dir}/data", exist_ok=True)
+    with open(f"{output_dir}/project.json", "r") as f:
+        project_json = json.load(f)
+    with open(f"{dist_dir}/data/project.json", "w") as f:
+        f.write(json.dumps({"project_json": project_json}))
+
+    trace_dirs = glob(f"{output_dir}/*/", recursive=True)
+    for trace_dir in trace_dirs:
+        trace_name = os.path.basename(os.path.normpath(trace_dir))
+        if os.path.exists(f"{output_dir}/{trace_name}/data.json"):
+            os.makedirs(f"{dist_dir}/data/{trace_name}", exist_ok=True)
+            shutil.copyfile(
+                f"{output_dir}/{trace_name}/data.json",
+                f"{dist_dir}/data/{trace_name}/data.json",
+            )
+
+    shutil.copytree(VIEWER_PATH, dist_dir, dirs_exist_ok=True)

visivo-1.0.20/visivo/commands/init.py

@@ -0,0 +1,16 @@
+import click
+
+
+@click.command()
+def init():
+    """
+    Enables a quick set up by writing your target & api credentials to an env file.
+    """
+    from visivo.logging.logger import Logger
+
+    Logger.instance().debug("Initializing")
+
+    from visivo.commands.init_phase import init_phase
+
+    init_phase()
+    Logger.instance().success("Done")