datajunction 0.0.2.dev0__tar.gz → 0.0.2.dev4__tar.gz

datajunction-0.0.2.dev4/Makefile

@@ -0,0 +1,13 @@
+check:
+	pdm run pre-commit run --all-files
+
+lint:
+	make check
+
+test:
+	pdm run pytest -n auto --cov=datajunction --cov-report term-missing -vv tests/ --doctest-modules datajunction --without-integration --without-slow-integration ${PYTEST_ARGS}
+
+dev-release:
+	hatch version dev
+	hatch build
+	hatch publish

{datajunction-0.0.2.dev0 → datajunction-0.0.2.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datajunction
-Version: 0.0.2.dev0
+Version: 0.0.2.dev4
 Summary: DataJunction client library for connecting to a DataJunction server
 Project-URL: repository, https://github.com/DataJunction/dj
 Author-email: DataJunction Authors <yian.shang@gmail.com>

{datajunction-0.0.2.dev0 → datajunction-0.0.2.dev4}/datajunction/__about__.py

@@ -2,4 +2,4 @@
 Version for Hatch
 """
 
-__version__ = "0.0.2.dev0"
+__version__ = "0.0.2.dev4"

{datajunction-0.0.2.dev0 → datajunction-0.0.2.dev4}/datajunction/cli.py

@@ -22,22 +22,11 @@ class DJCLI:
         self.builder_client = builder_client or DJBuilder()
         self.deployment_service = DeploymentService(client=self.builder_client)
 
-    def deploy(self, directory: str, dryrun: bool):
-        """
-        Deploy nodes from the specified directory.
-        """
-        project = Project.load(directory)
-        compiled_project = project.compile()
-        if dryrun:
-            compiled_project.validate(client=self.builder_client)
-        else:
-            compiled_project.deploy(client=self.builder_client)
-
-    def push(self, directory: str):
+    def push(self, directory: str, namespace: str | None = None):
         """
         Alias for deploy without dryrun.
         """
-        self.deployment_service.push(directory)
+        self.deployment_service.push(directory, namespace=namespace)
 
     def pull(self, namespace: str, directory: str):
         """
@@ -79,6 +68,12 @@ class DJCLI:
             "directory",
             help="Path to the directory containing YAML files",
         )
+        push_parser.add_argument(
+            "--namespace",
+            type=str,
+            default=None,
+            help="The namespace to push to (optionally overrides the namespace in the YAML files)",
+        )
 
         # `dj pull <namespace> <directory>`
         pull_parser = subparsers.add_parser(
@@ -105,10 +100,8 @@ class DJCLI:
         """
         Dispatches the command based on the parsed args
         """
-        if args.command == "deploy":
-            self.deploy(args.directory, args.dryrun)
-        elif args.command == "push":
-            self.push(args.directory)
+        if args.command == "push":
+            self.push(args.directory, namespace=args.namespace)
         elif args.command == "pull":
             self.pull(args.namespace, args.directory)
         elif args.command == "seed":

{datajunction-0.0.2.dev0 → datajunction-0.0.2.dev4}/datajunction/deployment.py

@@ -96,7 +96,7 @@ class DeploymentService:
     def build_table(deployment_uuid: str, data: dict) -> Table:
         """Return a fresh Table with current deployment results."""
         table = Table(
-            title=f"Deployment {deployment_uuid}",
+            title=f"Deployment [bold green]{deployment_uuid}[/ bold green]\nNamespace [bold green]{data['namespace']}[/ bold green]",
             box=box.SIMPLE_HEAVY,
             expand=True,
         )
@@ -106,19 +106,28 @@ class DeploymentService:
         table.add_column("Status", style="green")
         table.add_column("Message", style="white")
 
+        color_mapping = {
+            "success": "bold green",
+            "failed": "bold red",
+            "pending": "yellow",
+            "skipped": "bold gray",
+        }
+
         for result in data.get("results", []):
+            color = color_mapping.get(result.get("status"), "white")
             table.add_row(
                 str(result.get("deploy_type", "")),
                 str(result.get("name", "")),
                 str(result.get("operation", "")),
-                str(result.get("status", "")),
-                str(result.get("message", "")),
+                f"[{color}]{result.get('status', '')}[/{color}]",
+                f"[gray]{result.get('message', '')}[/gray]",
             )
         return table
 
     def push(
         self,
         source_path: str | Path,
+        namespace: str | None = None,
         console: Console = Console(),
     ):
         """
@@ -127,12 +136,13 @@ class DeploymentService:
         console.print(f"[bold]Pushing project from:[/bold] {source_path}")
 
         deployment_spec = self._reconstruct_deployment_spec(source_path)
+        deployment_spec["namespace"] = namespace or deployment_spec.get("namespace")
         deployment_data = self.client.deploy(deployment_spec)
         deployment_uuid = deployment_data["uuid"]
 
-        console.print(f"[bold]Deployment initiated:[/bold] UUID {deployment_uuid}\n")
+        # console.print(f"[bold]Deployment initiated:[/bold] UUID {deployment_uuid}\n")
 
-        # Wait for deployment to finish
+        # Max wait time for deployment to finish
         timeout = time.time() + 300  # 5 minutes
 
         with Live(
@@ -152,8 +162,9 @@ class DeploymentService:
                     raise DJClientException("Deployment timed out after 5 minutes")
 
             live.update(DeploymentService.build_table(deployment_uuid, deployment_data))
+            color = "green" if deployment_data.get("status") == "success" else "red"
             console.print(
-                f"\n[bold]Deployment finished:[/bold] {deployment_data.get('status')}",
+                f"\nDeployment finished: [bold {color}]{deployment_data.get('status').upper()}[/bold {color}]",
             )
 
     @staticmethod

{datajunction-0.0.2.dev0 → datajunction-0.0.2.dev4}/tests/test_cli.py

@@ -13,39 +13,6 @@ from datajunction import DJBuilder
 from datajunction.cli import main
 
 
-def test_deploy(change_to_project_dir: Callable):
-    """
-    Test `dj deploy <dir>`
-    """
-    builder_client = mock.MagicMock()
-
-    # Test deploy with dryrun
-    change_to_project_dir("./")
-    test_args = ["dj", "deploy", "./project9", "--dryrun"]
-    with patch.object(sys, "argv", test_args):
-        main(builder_client=builder_client)
-
-    func_names = [mock_call[0] for mock_call in builder_client.mock_calls]
-    assert "basic_login" in func_names
-    assert "create_namespace" in func_names
-    assert "create_source" in func_names
-    assert "create_dimension" in func_names
-    assert "delete_namespace" in func_names
-
-    # Test deploy without dryrun
-    change_to_project_dir("./")
-    test_args = ["dj", "deploy", "./project9"]
-    with patch.object(sys, "argv", test_args):
-        main(builder_client=builder_client)
-
-    func_names = [mock_call[0] for mock_call in builder_client.mock_calls]
-    assert "basic_login" in func_names
-    assert "create_namespace" in func_names
-    assert "create_source" in func_names
-    assert "create_dimension" in func_names
-    assert "delete_namespace" in func_names
-
-
 def test_pull(
     tmp_path,
     builder_client: DJBuilder,  # pylint: disable=redefined-outer-name
@@ -84,6 +51,16 @@ def test_push_full(
     results = builder_client.list_nodes(namespace="deps.deploy0")
     assert len(results) == 6
 
+    test_args = ["dj", "push", "./deploy0", "--namespace", "deps.deploy0.main"]
+    with patch.dict(os.environ, env_vars, clear=False):
+        with patch.object(sys, "argv", test_args):
+            main(builder_client=builder_client)
+
+    results = builder_client.list_nodes(namespace="deps.deploy0.main")
+    assert len(results) == 6
+    results = builder_client.list_nodes(namespace="deps.deploy0")
+    assert len(results) == 12
+
 
 def test_seed():
     """

{datajunction-0.0.2.dev0 → datajunction-0.0.2.dev4}/tests/test_compile.py

@@ -294,11 +294,11 @@ def test_compile_deploying_a_project(
     assert [
         link["dimension"]["name"] for link in local_hard_hats["dimension_links"]
     ] == ["projects.project1.roads.us_state"]
-    assert [
-        col["dimension"]["name"]
-        for col in local_hard_hats["columns"]
-        if col["name"] == "birth_date"
-    ] == ["projects.project1.roads.date_dim"]
+    # assert [
+    #     col["dimension"]["name"]
+    #     for col in local_hard_hats["columns"]
+    #     if col["name"] == "birth_date"
+    # ] == ["projects.project1.roads.date_dim"]
 
     # Check metric metadata and required dimensions
     avg_repair_price = builder_client.metric("projects.project1.roads.avg_repair_price")
@@ -363,7 +363,7 @@ def test_compile_raising_on_invalid_table_name(
     project = Project.load_current()
     with pytest.raises(DJClientException) as exc_info:
         project.compile()
-    assert "Invalid table name" in str(exc_info.value)
+    assert "Invalid" in str(exc_info.value)
 
 
 def test_compile_raising_on_invalid_file_name(

datajunction-0.0.2.dev4/tests/test_deploy.py

@@ -0,0 +1,176 @@
+import io
+from pathlib import Path
+import time
+from unittest import mock
+import pytest
+from unittest.mock import MagicMock
+from datajunction.deployment import DeploymentService
+from datajunction.exceptions import DJClientException
+import yaml
+from rich.console import Console
+
+
+def test_clean_dict_removes_nones_and_empty():
+    dirty = {
+        "a": None,
+        "b": [],
+        "c": {},
+        "d": {"x": None, "y": {"z": []}, "k": "keep"},
+        "e": [1, 2],
+    }
+    cleaned = DeploymentService.clean_dict(dirty)
+    assert cleaned == {"d": {"k": "keep"}, "e": [1, 2]}
+
+
+def test_pull_writes_yaml_files(tmp_path):
+    # fake client returning a minimal deployment spec
+    client = MagicMock()
+    client._export_namespace_spec.return_value = {
+        "namespace": "foo.bar",
+        "nodes": [
+            {"name": "foo.bar.baz", "query": "SELECT 1"},
+            {"name": "foo.bar.qux", "query": "SELECT 2"},
+        ],
+    }
+    svc = DeploymentService(client)
+
+    svc.pull("foo.bar", tmp_path)
+
+    # project-level yaml
+    project_yaml = yaml.safe_load((tmp_path / "dj.yaml").read_text())
+    assert project_yaml["namespace"] == "foo.bar"
+
+    # node files
+    baz_file = tmp_path / "foo" / "bar" / "baz.yaml"
+    assert baz_file.exists()
+    assert yaml.safe_load(baz_file.read_text())["query"] == "SELECT 1"
+
+    qux_file = tmp_path / "foo" / "bar" / "qux.yaml"
+    assert qux_file.exists()
+
+
+def test_pull_raises_if_target_not_empty(tmp_path):
+    (tmp_path / "something.txt").write_text("not empty")
+    client = MagicMock()
+    svc = DeploymentService(client)
+    with pytest.raises(DJClientException):
+        svc.pull("ns", tmp_path)
+
+
+def test_build_table_has_expected_columns():
+    tbl = DeploymentService.build_table(
+        "abc-123",
+        {
+            "namespace": "some.namespace",
+            "status": "success",
+            "results": [
+                {
+                    "deploy_type": "node",
+                    "name": "some.random.node",
+                    "operation": "create",
+                    "status": "success",
+                    "message": "ok",
+                },
+            ],
+        },
+    )
+    cols = [c.header for c in tbl.columns]
+    assert cols == ["Type", "Name", "Operation", "Status", "Message"]
+    row_values = [col._cells[0] for col in tbl.columns]
+    assert row_values == [
+        "node",
+        "some.random.node",
+        "create",
+        "[bold green]success[/bold green]",
+        "[gray]ok[/gray]",
+    ]
+
+
+def test_reconstruct_deployment_spec(tmp_path):
+    # set up a fake exported project
+    (tmp_path / "dj.yaml").write_text(
+        yaml.safe_dump({"namespace": "foo", "tags": ["t1"]}),
+    )
+    node_dir = tmp_path / "foo"
+    node_dir.mkdir()
+    node_file = node_dir / "bar.yaml"
+    node_file.write_text(yaml.safe_dump({"name": "foo.bar", "query": "SELECT 1"}))
+
+    svc = DeploymentService(MagicMock())
+    spec = svc._reconstruct_deployment_spec(tmp_path)
+    assert spec["namespace"] == "foo"
+    assert spec["tags"] == ["t1"]
+    assert spec["nodes"][0]["name"] == "foo.bar"
+
+
+@pytest.mark.timeout(2)
+def test_push_waits_until_success(monkeypatch, tmp_path):
+    # Create a fake project structure so _reconstruct_deployment_spec returns something
+    (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "foo"}))
+    (tmp_path / "foo.yaml").write_text(yaml.safe_dump({"name": "foo.bar"}))
+
+    # Fake client that returns "pending" once then "success"
+    client = MagicMock()
+    responses = [
+        {"uuid": "123", "status": "pending", "results": [], "namespace": "foo"},
+        {"uuid": "123", "status": "success", "results": [], "namespace": "foo"},
+    ]
+    client.deploy.return_value = responses[0]
+    client.check_deployment.side_effect = responses[1:]
+
+    svc = DeploymentService(client, console=Console(file=io.StringIO()))
+    monkeypatch.setattr(time, "sleep", lambda _: None)
+
+    svc.push(tmp_path)  # should not raise
+
+    client.deploy.assert_called_once()
+    client.check_deployment.assert_called()
+
+
+def test_push_times_out(monkeypatch, tmp_path):
+    # minimal project structure so _reconstruct_deployment_spec works
+    (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "foo"}))
+    (tmp_path / "foo.yaml").write_text(yaml.safe_dump({"name": "foo.bar"}))
+
+    # Fake client: deploy returns a uuid, check_deployment always 'pending'
+    client = MagicMock()
+    client.deploy.return_value = {
+        "uuid": "123",
+        "status": "pending",
+        "results": [],
+        "namespace": "foo",
+    }
+    client.check_deployment.return_value = {
+        "uuid": "123",
+        "status": "pending",
+        "results": [],
+        "namespace": "foo",
+    }
+
+    svc = DeploymentService(client, console=Console(file=io.StringIO()))
+
+    # Patch time.sleep to skip waiting
+    monkeypatch.setattr(time, "sleep", lambda _: None)
+
+    # Simulate time moving past the timeout on the second call
+    start_time = 1_000_000
+    times = [start_time, start_time + 301]  # second call is > 5 minutes later
+    monkeypatch.setattr(time, "time", lambda: times.pop(0))
+
+    with pytest.raises(DJClientException, match="Deployment timed out"):
+        svc.push(tmp_path)
+
+    client.deploy.assert_called_once()
+    client.check_deployment.assert_called()
+
+
+def test_read_project_yaml_returns_empty(tmp_path: Path):
+    """
+    Verify that when dj.yaml is missing, _read_project_yaml returns an empty dict.
+    """
+    # Create a directory without dj.yaml
+    project_dir = tmp_path
+    client = mock.MagicMock()
+    svc = DeploymentService(client, console=Console(file=io.StringIO()))
+    result = svc._read_project_yaml(project_dir)
+    assert result == {}

datajunction-0.0.2.dev0/Makefile

@@ -1,13 +0,0 @@
-check:
-	pdm run pre-commit run --all-files
-
-lint:
-	make check
-
-test:
-	pdm run pytest -n auto --cov=datajunction --cov-report=term -vv tests/ --doctest-modules datajunction --without-integration --without-slow-integration ${PYTEST_ARGS}
-
-dev-release:
-	hatch version dev
-	hatch build
-	hatch publish

datajunction-0.0.2.dev0/target/dj.yaml

@@ -1,3 +0,0 @@
-name: Project users.yshang.member (Autogenerated)
-description: This is an autogenerated project for namespace users.yshang.member
-namespace: users.yshang.member0

datajunction-0.0.2.dev0/target/draft_ms.yaml

@@ -1,86 +0,0 @@
-name: ${prefix}draft_ms
-node_type: cube
-owners:
-- dj
-display_name: Draft MS
-description: ''
-mode: published
-metrics:
-- ${prefix}total_missing_covariates
-- ${prefix}overall_streaming_lift
-- ${prefix}engagement_quality_score
-- ${prefix}qpd_normalization__35
-dimensions:
-- common.dimensions.client_category.client_name
-- common.dimensions.hardware_category.hw_category
-- common.dimensions.xp.ab_test_plan.group_id
-- common.dimensions.xp.allocation_day.alloc_country_iso_code
-- common.dimensions.xp.measure_date.dateint
-- member.is_jfk_profile.is_jfk_profile
-columns:
-- name: users.yshang.member.total_missing_covariates
-  type: int
-  display_name: Total Missing Covariates
-  description: null
-  attributes: []
-  partition: null
-- name: users.yshang.member.overall_streaming_lift
-  type: float
-  display_name: Overall Streaming Lift
-  description: null
-  attributes: []
-  partition: null
-- name: users.yshang.member.engagement_quality_score
-  type: double
-  display_name: Engagement Quality Score
-  description: null
-  attributes: []
-  partition: null
-- name: users.yshang.member.qpd_normalization__35
-  type: float
-  display_name: QPD (normalization = 35)
-  description: null
-  attributes: []
-  partition: null
-- name: common.dimensions.client_category.client_name
-  type: string
-  display_name: Client Name
-  description: null
-  attributes:
-  - primary_key
-  partition: null
-- name: common.dimensions.hardware_category.hw_category
-  type: string
-  display_name: Hw Category
-  description: null
-  attributes:
-  - primary_key
-  partition: null
-- name: common.dimensions.xp.ab_test_plan.group_id
-  type: string
-  display_name: Allocation Plan ID
-  description: null
-  attributes:
-  - primary_key
-  partition: null
-- name: common.dimensions.xp.allocation_day.alloc_country_iso_code
-  type: string
-  display_name: Alloc Country Iso Code
-  description: null
-  attributes: []
-  partition: null
-- name: common.dimensions.xp.measure_date.dateint
-  type: int
-  display_name: Dateint
-  description: null
-  attributes:
-  - primary_key
-  - temporal
-  partition: null
-- name: member.is_jfk_profile.is_jfk_profile
-  type: int
-  display_name: Is Jfk Profile
-  description: null
-  attributes:
-  - primary_key
-  partition: null

datajunction-0.0.2.dev0/target/draft_ms_with_qpd.yaml

@@ -1,78 +0,0 @@
-name: ${prefix}draft_ms_with_qpd
-node_type: cube
-owners:
-- yshang@netflix.com
-display_name: draft MS with QPD
-description: ''
-mode: published
-metrics:
-- ${prefix}total_missing_covariates
-- ${prefix}overall_streaming_lift
-- ${prefix}qpd
-- ${prefix}engagement_quality_score
-- ${prefix}qpd_normalization__35
-dimensions:
-- common.dimensions.xp.measure_date.dateint
-- member.is_jfk_profile.is_jfk_profile
-- common.dimensions.hardware_category.hw_category
-- common.dimensions.client_category.client_name
-columns:
-- name: users.yshang.member.total_missing_covariates
-  type: int
-  display_name: Total Missing Covariates
-  description: null
-  attributes: []
-  partition: null
-- name: users.yshang.member.overall_streaming_lift
-  type: float
-  display_name: Overall Streaming Lift
-  description: null
-  attributes: []
-  partition: null
-- name: users.yshang.member.qpd
-  type: float
-  display_name: Video QPD per 28d
-  description: null
-  attributes: []
-  partition: null
-- name: users.yshang.member.engagement_quality_score
-  type: double
-  display_name: Engagement Quality Score
-  description: null
-  attributes: []
-  partition: null
-- name: users.yshang.member.qpd_normalization__35
-  type: float
-  display_name: QPD (normalization = 35)
-  description: null
-  attributes: []
-  partition: null
-- name: common.dimensions.xp.measure_date.dateint
-  type: int
-  display_name: Dateint
-  description: null
-  attributes:
-  - temporal
-  - primary_key
-  partition: null
-- name: member.is_jfk_profile.is_jfk_profile
-  type: int
-  display_name: Is Jfk Profile
-  description: null
-  attributes:
-  - primary_key
-  partition: null
-- name: common.dimensions.hardware_category.hw_category
-  type: string
-  display_name: Hw Category
-  description: null
-  attributes:
-  - primary_key
-  partition: null
-- name: common.dimensions.client_category.client_name
-  type: string
-  display_name: Client Name
-  description: null
-  attributes:
-  - primary_key
-  partition: null

datajunction-0.0.2.dev0/target/engagement_quality_score.yaml

@@ -1,12 +0,0 @@
-name: ${prefix}engagement_quality_score
-node_type: metric
-owners:
-- yshang@netflix.com
-display_name: Engagement Quality Score
-description: ''
-mode: published
-query: |-
-  SELECT CAST(SUM(CASE WHEN wsum_eqs_v3_score_pts IS NOT NULL THEN wsum_eqs_v3_score_pts ELSE NULL END) AS DOUBLE) /
-  SUM(CASE WHEN wsum_eqs_v3_score_pts IS NOT NULL THEN eqs_view_secs ELSE NULL END) FROM ${prefix}playbacks
-direction: higher_is_better
-significant_digits: 3

datajunction-0.0.2.dev0/target/overall_streaming_lift.yaml

@@ -1,8 +0,0 @@
-name: ${prefix}overall_streaming_lift
-node_type: metric
-owners:
-- yshang@netflix.com
-display_name: Overall Streaming Lift
-description: ''
-mode: published
-query: SELECT CAST(SUM(view_secs / 3600.0) AS FLOAT) FROM ${prefix}playbacks