data-designer 0.1.3__tar.gz → 0.1.4__tar.gz

data_designer-0.1.4/.github/workflows/build-docs.yml

@@ -0,0 +1,39 @@
+name: Build docs
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  build-notebooks:
+    uses: ./.github/workflows/build-notebooks.yml
+    secrets: inherit
+  deploy:
+    needs: build-notebooks
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.9.5"
+      - name: Set up Python
+        run: uv python install
+      - name: Install dependencies for docs
+        run: uv sync --group docs
+      - name: Download artifact from previous step
+        uses: actions/download-artifact@v5
+        with:
+          name: notebooks
+          path: docs/notebooks
+      - name: Extract version from release
+        run: |
+          # Remove the 'v' prefix and any suffix after a space
+          VERSION=$(echo ${{ github.event.release.tag_name }} | sed 's/^v//' | sed 's/ .*$//')
+          echo "::notice::Extracted version: $VERSION"
+          echo "VERSION=$VERSION" >> $GITHUB_ENV
+      - name: Build and deploy docs
+        run: uv run mike deploy --push --update-aliases ${{ env.VERSION }} latest

data_designer-0.1.4/.github/workflows/build-notebooks.yml

@@ -0,0 +1,30 @@
+name: Build notebooks
+on:
+  workflow_call:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * MON"
+  
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.9.5"
+      - name: Set up Python
+        run: uv python install 3.11
+      - name: Convert and execute notebooks
+        run: make convert-execute-notebooks
+      - name: Upload notebooks as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: notebooks
+          path: docs/notebooks

{data_designer-0.1.3 → data_designer-0.1.4}/.github/workflows/pack-tutorials.yml

@@ -1,4 +1,4 @@
-name: Pack Tutorials
+name: Pack tutorials
 
 on:
   workflow_dispatch:
@@ -6,20 +6,26 @@ on:
     types: [published]
 
 jobs:
-  zip_and_upload:
+  build-notebooks:
+    uses: ./.github/workflows/build-notebooks.yml
+    secrets: inherit
+  zip-and-upload:
+    needs: build-notebooks
     runs-on: ubuntu-latest
     permissions:
       contents: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
+      - name: Download artifact from previous step
+        uses: actions/download-artifact@v5
+        with:
+          name: notebooks
+          path: data_designer_tutorial
 
       - name: Set up environment variables
         id: env_setup
         run: |
-          echo "SOURCE_FOLDER_PATH=docs/notebooks" >> $GITHUB_ENV
-          echo "TARGET_FOLDER_NAME=data_designer_tutorial" >> $GITHUB_ENV
+          echo "SOURCE_FOLDER_PATH=data_designer_tutorial" >> $GITHUB_ENV
           echo "ZIP_FILE_NAME=data_designer_tutorial.zip" >> $GITHUB_ENV
           
       - name: Check if source folder exists
@@ -29,12 +35,9 @@ jobs:
             exit 1
           fi
 
-      - name: Rename source folder
-        run: mv ${{ env.SOURCE_FOLDER_PATH }} ${{ env.TARGET_FOLDER_NAME }}
-
       - name: Zip the target folder
         run: |
-          zip -r ${{ env.ZIP_FILE_NAME }} ${{ env.TARGET_FOLDER_NAME }}
+          zip -r ${{ env.ZIP_FILE_NAME }} ${{ env.SOURCE_FOLDER_PATH }}
           echo "Successfully created ${{ env.ZIP_FILE_NAME }}"
 
       - name: Find the latest existing release tag

{data_designer-0.1.3 → data_designer-0.1.4}/.gitignore

@@ -87,4 +87,7 @@ src/data_designer/_version.py
 
 .claude/
 
-docs/examples/artifacts/
+docs/notebooks/
+docs/notebook_source/*.ipynb
+docs/notebook_source/*.csv
+docs/notebook_source/artifacts/

{data_designer-0.1.3 → data_designer-0.1.4}/Makefile

@@ -15,29 +15,30 @@ help:
 	@echo "═════════════════════════════════════════════════════════════"
 	@echo ""
 	@echo "📦 Installation:"
-	@echo "  install                - Install project dependencies with uv"
-	@echo "  install-dev            - Install project with dev dependencies"
-	@echo "  install-dev-notebooks  - Install dev + notebook dependencies (Jupyter, etc.)"
+	@echo "  install                   - Install project dependencies with uv"
+	@echo "  install-dev               - Install project with dev dependencies"
+	@echo "  install-dev-notebooks     - Install dev + notebook dependencies (Jupyter, etc.)"
 	@echo ""
 	@echo "🧪 Testing:"
-	@echo "  test                   - Run all unit tests"
-	@echo "  coverage               - Run tests with coverage report"
+	@echo "  test                      - Run all unit tests"
+	@echo "  coverage                  - Run tests with coverage report"
 	@echo ""
 	@echo "✨ Code Quality:"
-	@echo "  format                 - Format code with ruff"
-	@echo "  format-check           - Check code formatting without making changes"
-	@echo "  lint                   - Lint code with ruff"
-	@echo "  lint-fix               - Fix linting issues automatically"
+	@echo "  format                    - Format code with ruff"
+	@echo "  format-check              - Check code formatting without making changes"
+	@echo "  lint                      - Lint code with ruff"
+	@echo "  lint-fix                  - Fix linting issues automatically"
 	@echo ""
 	@echo "🔍 Combined Checks:"
-	@echo "  check-all              - Run all checks (format-check + lint)"
-	@echo "  check-all-fix          - Run all checks with autofix (format + lint-fix)"
+	@echo "  check-all                 - Run all checks (format-check + lint)"
+	@echo "  check-all-fix             - Run all checks with autofix (format + lint-fix)"
 	@echo ""
 	@echo "🛠️  Utilities:"
-	@echo "  clean                  - Remove coverage reports and cache files"
-	@echo "  serve-docs-locally     - Serve documentation locally"
-	@echo "  check-license-headers  - Check if all files have license headers"
-	@echo "  update-license-headers - Add license headers to all files"
+	@echo "  clean                     - Remove coverage reports and cache files"
+	@echo "  convert-execute-notebooks - Convert notebooks from .py to .ipynb using jupytext"
+	@echo "  serve-docs-locally        - Serve documentation locally"
+	@echo "  check-license-headers     - Check if all files have license headers"
+	@echo "  update-license-headers    - Add license headers to all files"
 	@echo ""
 	@echo "═════════════════════════════════════════════════════════════"
 	@echo "💡 Tip: Run 'make <command>' to execute any command above"
@@ -83,6 +84,17 @@ test:
 	@echo "🧪 Running unit tests..."
 	uv run --group dev pytest
 
+convert-execute-notebooks:
+	@echo "📓 Converting Python tutorials to notebooks and executing..."
+	@mkdir -p docs/notebooks
+	cp docs/notebook_source/_README.md docs/notebooks/README.md
+	cp docs/notebook_source/_pyproject.toml docs/notebooks/pyproject.toml
+	uv run --group notebooks --group docs jupytext --to ipynb --execute docs/notebook_source/*.py
+	mv docs/notebook_source/*.ipynb docs/notebooks/
+	rm -r docs/notebook_source/artifacts
+	rm docs/notebook_source/*.csv
+	@echo "✅ Notebooks created in docs/notebooks/"
+
 serve-docs-locally:
 	@echo "📝 Building and serving docs..."
 	uv sync --group docs

{data_designer-0.1.3 → data_designer-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-designer
-Version: 0.1.3
+Version: 0.1.4
 Summary: General framework for synthetic data generation
 License-Expression: Apache-2.0
 License-File: LICENSE
@@ -148,7 +148,7 @@ preview.display_sample_record()
 - **[Tutorial Notebooks](https://nvidia-nemo.github.io/DataDesigner/notebooks/)** – Step-by-step interactive tutorials
 - **[Column Types](https://nvidia-nemo.github.io/DataDesigner/concepts/columns/)** – Explore samplers, LLM columns, validators, and more
 - **[Validators](https://nvidia-nemo.github.io/DataDesigner/concepts/validators/)** – Learn how to validate generated data with Python, SQL, and remote validators
-- **[Model Configuration](https://nvidia-nemo.github.io/DataDesigner/models/model-configs/)** – Configure custom models and providers
+- **[Model Configuration](https://nvidia-nemo.github.io/DataDesigner/concepts/models/model-configs/)** – Configure custom models and providers
 - **[Person Sampling](https://nvidia-nemo.github.io/DataDesigner/concepts/person_sampling/)** – Learn how to sample realistic person data with demographic attributes
 
 ### 🔧 Configure models via CLI

{data_designer-0.1.3 → data_designer-0.1.4}/README.md

@@ -99,7 +99,7 @@ preview.display_sample_record()
 - **[Tutorial Notebooks](https://nvidia-nemo.github.io/DataDesigner/notebooks/)** – Step-by-step interactive tutorials
 - **[Column Types](https://nvidia-nemo.github.io/DataDesigner/concepts/columns/)** – Explore samplers, LLM columns, validators, and more
 - **[Validators](https://nvidia-nemo.github.io/DataDesigner/concepts/validators/)** – Learn how to validate generated data with Python, SQL, and remote validators
-- **[Model Configuration](https://nvidia-nemo.github.io/DataDesigner/models/model-configs/)** – Configure custom models and providers
+- **[Model Configuration](https://nvidia-nemo.github.io/DataDesigner/concepts/models/model-configs/)** – Configure custom models and providers
 - **[Person Sampling](https://nvidia-nemo.github.io/DataDesigner/concepts/person_sampling/)** – Learn how to sample realistic person data with demographic attributes
 
 ### 🔧 Configure models via CLI

data_designer-0.1.4/docs/code_reference/models.md

@@ -0,0 +1,11 @@
+# Models
+
+The `models` module defines configuration objects for model-based generation. [ModelProvider](#data_designer.config.models.ModelProvider), specifies connection and authentication details for custom providers. [ModelConfig](#data_designer.config.models.ModelConfig) encapsulates model details including the model alias, identifier, and inference parameters. [InferenceParameters](#data_designer.config.models.InferenceParameters) controls model behavior through settings like `temperature`, `top_p`, and `max_tokens`, with support for both fixed values and distribution-based sampling. The module includes [ImageContext](#data_designer.config.models.ImageContext) for providing image inputs to multimodal models.
+
+For more information on how they are used, see below:
+
+- **[Model Providers](../concepts/models/model-providers.md)**
+- **[Model Configs](../concepts/models/model-configs.md)**
+- **[Image Context](/notebooks/4-providing-images-as-context/)**
+
+::: data_designer.config.models

{data_designer-0.1.3/docs → data_designer-0.1.4/docs/concepts}/models/configure-model-settings-with-the-cli.md

@@ -132,4 +132,4 @@ The CLI will show which configuration files exist and ask for confirmation befor
 - **[Model Providers](model-providers.md)**: Learn about the `ModelProvider` class and provider configuration
 - **[Model Configurations](model-configs.md)**: Learn about `ModelConfig` and `InferenceParameters`
 - **[Default Model Settings](default-model-settings.md)**: Pre-configured providers and model settings included with Data Designer
-- **[Quick Start Guide](../quick-start.md)**: Get started with a simple example
+- **[Quick Start Guide](../../quick-start.md)**: Get started with a simple example

{data_designer-0.1.3/docs → data_designer-0.1.4/docs/concepts}/models/default-model-settings.md

@@ -91,5 +91,5 @@ Both methods operate on the same files, ensuring consistency across your entire
 ## See Also
 
 - **[Configure Model Settings With the CLI](configure-model-settings-with-the-cli.md)**: Learn how to use the CLI to manage model settings.
-- **[Quick Start Guide](../quick-start.md)**: Get started with a simple example
+- **[Quick Start Guide](../../quick-start.md)**: Get started with a simple example
 - **[Model Configurations](model-configs.md)**: Learn about model configurations

{data_designer-0.1.3/docs → data_designer-0.1.4/docs/concepts}/models/model-configs.md

@@ -241,4 +241,4 @@ model_config = ModelConfig(
 - **[Model Providers](model-providers.md)**: Learn about configuring model providers
 - **[Default Model Settings](default-model-settings.md)**: Pre-configured model settings included with Data Designer
 - **[Configure Model Settings With the CLI](configure-model-settings-with-the-cli.md)**: Use the CLI to manage model settings
-- **[Column Configurations](../code_reference/column_configs.md)**: Learn how to use models in column configurations
+- **[Column Configurations](../../code_reference/column_configs.md)**: Learn how to use models in column configurations

{data_designer-0.1.3/docs → data_designer-0.1.4/docs/concepts}/models/model-providers.md

@@ -47,4 +47,4 @@ provider = ModelProvider(
 - **[Model Configurations](model-configs.md)**: Learn about configuring models and inference parameters
 - **[Default Model Settings](default-model-settings.md)**: Pre-configured providers and model settings included with Data Designer
 - **[Configure Model Settings With the CLI](configure-model-settings-with-the-cli.md)**: Use the CLI to manage providers and model settings
-- **[Quick Start Guide](../quick-start.md)**: Get started with a simple example
+- **[Quick Start Guide](../../quick-start.md)**: Get started with a simple example

{data_designer-0.1.3 → data_designer-0.1.4}/docs/index.md

@@ -25,7 +25,7 @@ Data Designer helps you create datasets through an intuitive, **iterative** proc
 1.  **⚙️ Configure** your model settings
     - Bring your own OpenAI-compatible model providers and models
     - Or use the default model providers and models to get started quickly
-    - Learn more by reading the [model docs](models/default-model-settings.md)
+    - Learn more by reading the [model docs](concepts/models/default-model-settings.md)
 2.  **🏗️ Design** your dataset
     - Iteratively design your dataset, column by column
     - Leverage tools like statistical samplers and LLMs to generate a variety of data types

data_designer-0.1.4/docs/notebook_source/1-the-basics.py

@@ -0,0 +1,342 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.18.1
+#   kernelspec:
+#     display_name: .venv
+#     language: python
+#     name: python3
+# ---
+
+# %% [markdown]
+# # 🎨 Data Designer Tutorial: The Basics
+#
+# #### 📚 What you'll learn
+#
+# This notebook demonstrates the basics of Data Designer by generating a simple product review dataset.
+#
+
+# %% [markdown]
+# ### 📦 Import the essentials
+#
+# - The `essentials` module provides quick access to the most commonly used objects.
+#
+
+# %%
+from data_designer.essentials import (
+    CategorySamplerParams,
+    DataDesigner,
+    DataDesignerConfigBuilder,
+    InferenceParameters,
+    LLMTextColumnConfig,
+    ModelConfig,
+    PersonFromFakerSamplerParams,
+    SamplerColumnConfig,
+    SamplerType,
+    SubcategorySamplerParams,
+    UniformSamplerParams,
+)
+
+# %% [markdown]
+# ### ⚙️ Initialize the Data Designer interface
+#
+# - `DataDesigner` is the main object is responsible for managing the data generation process.
+#
+# - When initialized without arguments, the [default model providers](https://nvidia-nemo.github.io/DataDesigner/concepts/models/default-model-settings/) are used.
+#
+
+# %%
+data_designer = DataDesigner()
+
+# %% [markdown]
+# ### 🎛️ Define model configurations
+#
+# - Each `ModelConfig` defines a model that can be used during the generation process.
+#
+# - The "model alias" is used to reference the model in the Data Designer config (as we will see below).
+#
+# - The "model provider" is the external service that hosts the model (see the [model config](https://nvidia-nemo.github.io/DataDesigner/concepts/models/default-model-settings/) docs for more details).
+#
+# - By default, we use [build.nvidia.com](https://build.nvidia.com/models) as the model provider.
+#
+
+# %%
+# This name is set in the model provider configuration.
+MODEL_PROVIDER = "nvidia"
+
+# The model ID is from build.nvidia.com.
+MODEL_ID = "nvidia/nvidia-nemotron-nano-9b-v2"
+
+# We choose this alias to be descriptive for our use case.
+MODEL_ALIAS = "nemotron-nano-v2"
+
+# This sets reasoning to False for the nemotron-nano-v2 model.
+SYSTEM_PROMPT = "/no_think"
+
+model_configs = [
+    ModelConfig(
+        alias=MODEL_ALIAS,
+        model=MODEL_ID,
+        provider=MODEL_PROVIDER,
+        inference_parameters=InferenceParameters(
+            temperature=0.5,
+            top_p=1.0,
+            max_tokens=1024,
+        ),
+    )
+]
+
+# %% [markdown]
+# ### 🏗️ Initialize the Data Designer Config Builder
+#
+# - The Data Designer config defines the dataset schema and generation process.
+#
+# - The config builder provides an intuitive interface for building this configuration.
+#
+# - The list of model configs is provided to the builder at initialization.
+#
+
+# %%
+config_builder = DataDesignerConfigBuilder(model_configs=model_configs)
+
+# %% [markdown]
+# ## 🎲 Getting started with sampler columns
+#
+# - Sampler columns offer non-LLM based generation of synthetic data.
+#
+# - They are particularly useful for **steering the diversity** of the generated data, as we demonstrate below.
+#
+# <br>
+#
+# You can view available samplers using the config builder's `info` property:
+#
+
+# %%
+config_builder.info.display("samplers")
+
+# %% [markdown]
+# Let's start designing our product review dataset by adding product category and subcategory columns.
+#
+
+# %%
+config_builder.add_column(
+    SamplerColumnConfig(
+        name="product_category",
+        sampler_type=SamplerType.CATEGORY,
+        params=CategorySamplerParams(
+            values=[
+                "Electronics",
+                "Clothing",
+                "Home & Kitchen",
+                "Books",
+                "Home Office",
+            ],
+        ),
+    )
+)
+
+config_builder.add_column(
+    SamplerColumnConfig(
+        name="product_subcategory",
+        sampler_type=SamplerType.SUBCATEGORY,
+        params=SubcategorySamplerParams(
+            category="product_category",
+            values={
+                "Electronics": [
+                    "Smartphones",
+                    "Laptops",
+                    "Headphones",
+                    "Cameras",
+                    "Accessories",
+                ],
+                "Clothing": [
+                    "Men's Clothing",
+                    "Women's Clothing",
+                    "Winter Coats",
+                    "Activewear",
+                    "Accessories",
+                ],
+                "Home & Kitchen": [
+                    "Appliances",
+                    "Cookware",
+                    "Furniture",
+                    "Decor",
+                    "Organization",
+                ],
+                "Books": [
+                    "Fiction",
+                    "Non-Fiction",
+                    "Self-Help",
+                    "Textbooks",
+                    "Classics",
+                ],
+                "Home Office": [
+                    "Desks",
+                    "Chairs",
+                    "Storage",
+                    "Office Supplies",
+                    "Lighting",
+                ],
+            },
+        ),
+    )
+)
+
+config_builder.add_column(
+    SamplerColumnConfig(
+        name="target_age_range",
+        sampler_type=SamplerType.CATEGORY,
+        params=CategorySamplerParams(values=["18-25", "25-35", "35-50", "50-65", "65+"]),
+    )
+)
+
+# Optionally validate that the columns are configured correctly.
+config_builder.validate()
+
+# %% [markdown]
+# Next, let's add samplers to generate data related to the customer and their review.
+#
+
+# %%
+config_builder.add_column(
+    SamplerColumnConfig(
+        name="customer",
+        sampler_type=SamplerType.PERSON_FROM_FAKER,
+        params=PersonFromFakerSamplerParams(age_range=[18, 70], locale="en_US"),
+    )
+)
+
+config_builder.add_column(
+    SamplerColumnConfig(
+        name="number_of_stars",
+        sampler_type=SamplerType.UNIFORM,
+        params=UniformSamplerParams(low=1, high=5),
+        convert_to="int",  # Convert the sampled float to an integer.
+    )
+)
+
+config_builder.add_column(
+    SamplerColumnConfig(
+        name="review_style",
+        sampler_type=SamplerType.CATEGORY,
+        params=CategorySamplerParams(
+            values=["rambling", "brief", "detailed", "structured with bullet points"],
+            weights=[1, 2, 2, 1],
+        ),
+    )
+)
+
+config_builder.validate()
+
+# %% [markdown]
+# ## 🦜 LLM-generated columns
+#
+# - The real power of Data Designer comes from leveraging LLMs to generate text, code, and structured data.
+#
+# - When prompting the LLM, we can use Jinja templating to reference other columns in the dataset.
+#
+# - As we see below, nested json fields can be accessed using dot notation.
+#
+
+# %%
+config_builder.add_column(
+    LLMTextColumnConfig(
+        name="product_name",
+        prompt=(
+            "You are a helpful assistant that generates product names. DO NOT add quotes around the product name.\n\n"
+            "Come up with a creative product name for a product in the '{{ product_category }}' category, focusing "
+            "on products related to '{{ product_subcategory }}'. The target age range of the ideal customer is "
+            "{{ target_age_range }} years old. Respond with only the product name, no other text."
+        ),
+        system_prompt=SYSTEM_PROMPT,
+        model_alias=MODEL_ALIAS,
+    )
+)
+
+config_builder.add_column(
+    LLMTextColumnConfig(
+        name="customer_review",
+        prompt=(
+            "You are a customer named {{ customer.first_name }} from {{ customer.city }}, {{ customer.state }}. "
+            "You are {{ customer.age }} years old and recently purchased a product called {{ product_name }}. "
+            "Write a review of this product, which you gave a rating of {{ number_of_stars }} stars. "
+            "The style of the review should be '{{ review_style }}'."
+        ),
+        system_prompt=SYSTEM_PROMPT,
+        model_alias=MODEL_ALIAS,
+    )
+)
+
+config_builder.validate()
+
+# %% [markdown]
+# ### 🔁 Iteration is key – preview the dataset!
+#
+# 1. Use the `preview` method to generate a sample of records quickly.
+#
+# 2. Inspect the results for quality and format issues.
+#
+# 3. Adjust column configurations, prompts, or parameters as needed.
+#
+# 4. Re-run the preview until satisfied.
+#
+
+# %%
+preview = data_designer.preview(config_builder, num_records=2)
+
+# %%
+# Run this cell multiple times to cycle through the 2 preview records.
+preview.display_sample_record()
+
+# %%
+# The preview dataset is available as a pandas DataFrame.
+preview.dataset
+
+# %% [markdown]
+# ### 📊 Analyze the generated data
+#
+# - Data Designer automatically generates a basic statistical analysis of the generated data.
+#
+# - This analysis is available via the `analysis` property of generation result objects.
+#
+
+# %%
+# Print the analysis as a table.
+preview.analysis.to_report()
+
+# %% [markdown]
+# ### 🆙 Scale up!
+#
+# - Happy with your preview data?
+#
+# - Use the `create` method to submit larger Data Designer generation jobs.
+#
+
+# %%
+results = data_designer.create(config_builder, num_records=10)
+
+# %%
+# Load the generated dataset as a pandas DataFrame.
+dataset = results.load_dataset()
+
+dataset.head()
+
+# %%
+# Load the analysis results into memory.
+analysis = results.load_analysis()
+
+analysis.to_report()
+
+# %% [markdown]
+# ## ⏭️ Next Steps
+#
+# Now that you've seen the basics of Data Designer, check out the following notebooks to learn more about:
+#
+# - [Structured outputs and jinja expressions](/notebooks/2-structured-outputs-and-jinja-expressions/)
+#
+# - [Seeding synthetic data generation with an external dataset](/notebooks/3-seeding-with-a-dataset/)
+#