snowflake-data-migration-orchestrator 0.2.0__tar.gz

snowflake_data_migration_orchestrator-0.2.0/.coveragerc

@@ -0,0 +1,23 @@
+[run]
+source = src
+omit =
+    */__init__.py
+    */__version__.py
+
+[report]
+exclude_lines =
+    pragma: no cover
+    def __repr__
+    raise NotImplementedError
+    if __name__ == .__main__.:
+    pass
+    raise ImportError
+
+[html]
+directory = tests/outcome/unit-coverage-html
+
+[xml]
+output = tests/outcome/unit-coverage.xml
+
+[json]
+output = tests/outcome/unit-coverage.json

snowflake_data_migration_orchestrator-0.2.0/.dockerignore

@@ -0,0 +1,18 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+.hatch/
+*.egg-info/
+dist/
+build/
+.git/
+.gitignore
+tests/
+*.md
+!README.md
+.env
+.venv
+venv/
+scripts

snowflake_data_migration_orchestrator-0.2.0/.env.example

@@ -0,0 +1,13 @@
+# Snowflake Connection Configuration
+# Copy this file to .env and fill in your actual values
+
+# Required: Basic Snowflake credentials
+SNOWFLAKE_ACCOUNT=your_account
+SNOWFLAKE_USER=your_user
+SNOWFLAKE_PASSWORD=your_password
+
+# Optional: Connection settings
+SNOWFLAKE_WAREHOUSE=your_warehouse
+SNOWFLAKE_DATABASE=your_database
+SNOWFLAKE_SCHEMA=your_schema
+SNOWFLAKE_ROLE=your_role

snowflake_data_migration_orchestrator-0.2.0/.gitignore

@@ -0,0 +1,210 @@
+# DS_Store
+.DS_Store
+**/.DS_Store
+
+# version
+!__version__.py
+!**/__version__.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Sphinx documentation
+docs/build/
+docs/source/**/_autosummary
+
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# pytest outcome
+*/tests/outcome/
+tests/outcome/**
+test-results.xml
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+logs/
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+# PyPI configuration file
+.pypirc
+
+# ruff cache
+.ruff_cache
+
+# Requirements file generated by pip-compile
+requirements.txt
+
+# Data Exchange Agent
+data-exchange-agent/src/mocked_api/*
+.vscode
+
+# Rye lock files (not committed for library projects)
+requirements.lock
+requirements-dev.lock
+
+# E2E test session config (contains credentials)
+e2e-tests/config.yaml
+
+# Temporary files
+temp/

snowflake_data_migration_orchestrator-0.2.0/CONTRIBUTING.md

@@ -0,0 +1,388 @@
+# CONTRIBUTING
+
+## Code Organization
+Data Migration Orchestrator is organized in these main modules:
+- `utils`: Utils that can be imported from anywhere.
+- `cloud_core`: Core for Cloud operations, which includes definitions for Workflows and Tasks.
+- `data_migration_core`: Core for Data Migration, which includes platform-agnostic logic.
+- `data_migration_cloud`: Workflows Initializers, Task, and Task Handlers for Data Migration Workflows.  
+- `cloud_orchestrator`: The Orchestrator itself, which consumes the other modules to support the execution of Workflows of any kind.
+
+These modules follow a layered architecture with strict import rules. An arrow from module A to module B means that A can import from B. Colors represent the architecture layers:
+- 🔵 **Blue**: Orchestration layer (`cloud_orchestrator`)
+- 🟢 **Green**: Migration logic layer (`data_migration_cloud`)
+- 🟠 **Orange**: Core components layer (`cloud_core`, `data_migration_core`)
+- 🟣 **Purple**: Foundation layer (`utils`)
+
+![Module Dependencies](docs/diagrams/module-dependencies.png)
+
+## Commands
+
+The orchestrator CLI supports multiple subcommands. When no subcommand is given, `start` is assumed for backward compatibility.
+
+### Starting the Orchestrator
+```bash
+hatch run start
+```
+
+### Creating a Workflow
+Validate a workflow configuration file and insert it into the `WORKFLOW` table:
+```bash
+hatch run create-workflow <config-file> [--connection-name <name>] [--name <workflow-name>] [--source-platform <platform>]
+```
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `config_file` | *(required)* | Path to the JSON workflow configuration file |
+| `--connection-name` | env vars | Snowflake connection name (see [Connection Management](#connection-management)) |
+| `--name` | `MY_WORKFLOW` | Name for the workflow record |
+| `--source-platform` | `sqlserver` | Source platform (`sqlserver`, `redshift`) |
+
+The command reads `affinity` from the configuration file itself (the top-level `affinity` field).
+
+Example:
+```bash
+hatch run create-workflow workflow-config.json --connection-name MY_CONN --name prod_migration
+```
+
+### Linting
+Run linting with [ruff](https://docs.astral.sh/ruff/):
+```bash
+# Run lint checks (with auto-fix)
+hatch run linter:check
+```
+
+### Type Checking
+Run static type checking with [ty](https://docs.astral.sh/ty/) (extremely fast Python type checker from Astral):
+```bash
+# Check all source code
+hatch run types:check-ty
+
+# Check specific path
+hatch run types:check-ty tests
+
+# Watch mode - automatically re-checks on file changes
+hatch run types:watch-ty
+```
+
+You can also run type checking with mypy:
+```bash
+hatch run types:check
+```
+
+### Import Validation
+Check that you're not violating the rules of the dependencies between the main modules described in [Code Organization](#code-organization):
+```bash
+./scripts/validate_imports.py
+```
+
+### ty Diagnostics Baseline
+
+The project uses a **diagnostics baseline** to prevent type error regressions. The CI will fail if the number of `ty` diagnostics increases beyond the baseline.
+
+| Project | Baseline |
+|---------|----------|
+| data-migration-orchestrator | 35 |
+
+To check diagnostics locally:
+```bash
+python .github/scripts/ty_check_diagnostics.py data-migration-orchestrator
+```
+
+If you **fix** type errors and reduce the count, please update the baseline in:
+- `.github/workflows/data-migration-orchestrator-all-ci.yml`
+- `.github/scripts/ty_check_diagnostics.py` (DEFAULT_BASELINES)
+
+### Testing
+Run the test suite:
+```bash
+# Run all tests with coverage
+hatch run test_all.py3.11:check
+
+# Run tests only
+hatch run test_all.py3.11:unit
+```
+
+### CI Workflows
+
+The following checks run automatically on PRs and pushes to `main`/`develop`:
+
+1. **Linting** - Static analysis with ruff
+2. **Type Check** - ty diagnostics baseline check
+3. **Unit Tests** - pytest on Python 3.11 and 3.12 with coverage
+
+## Docker
+
+### Commands
+Build image:
+```bash
+hatch run docker-build
+```
+
+Alternatively, build directly with Docker from the **parent directory** (required due to shared module dependencies):
+```bash
+# From the migrations-data-validation directory (parent)
+docker build -f data-migration-orchestrator/Dockerfile -t data-migration-orchestrator:latest .
+```
+
+Run container:
+```bash
+hatch run docker-run
+```
+
+Interactive mode (debugging):
+```bash
+hatch run docker-run-interactive
+```
+
+### Setup
+1. Create environment file from template:
+   ```bash
+   cp .env.example .env
+   ```
+
+2. Edit `.env` with your Snowflake credentials:
+   ```env
+   SNOWFLAKE_ACCOUNT=your_account
+   SNOWFLAKE_USER=your_user
+   SNOWFLAKE_PASSWORD=your_password
+   SNOWFLAKE_WAREHOUSE=your_warehouse
+   SNOWFLAKE_DATABASE=your_database
+   SNOWFLAKE_SCHEMA=your_schema
+   ```
+
+3. Build and run:
+   ```bash
+   hatch run docker-build
+   hatch run docker-run
+   ```
+
+### Connection Management
+The application automatically detects the environment:
+
+- **Local**: Uses credentials from `.env` file (via `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, `SNOWFLAKE_PASSWORD`, etc.)
+- **SPCS**: Automatically uses Snowflake-injected credentials (OAuth)
+
+No code changes needed when deploying to Snowflake Container Services (SPCS).
+
+## End-to-End Flow
+For testing the end-to-end flow, with one **Data Migration Orchestrator** running and one (or more) **Data Exchange Agents**, we have the following options:
+1. Running all components locally.
+2. Running all components in SPCS.
+3. Mixed approach: running some components locally and other SPCS.
+
+### Running all components locally
+#### Starting the Data Migration Orchestrator
+These steps will start the **Data Migration Orchestrator** locally, and when that is orchestrated, the corresponding resources in the Snowflake account will also be set up. That will allow you to start **Data Exchange Agents** in the same account.
+*<snowflake-connection-name>* must be the name of the connection you want to use, as it appears on your `config.toml`/`connections.toml`.
+1. Go to the `data-migration-orchestrator` directory (`cd data-migration-orchestrator`).
+2. `export SNOWFLAKE_CONNECTION_NAME=<snowflake-connection-name>`.
+3. `export DATA_MIGRATION_ORCHESTRATOR_AFFINITY=<affinity>` (this is optional, but is recommended for local development, to avoid confusion with multiple orchestrator instances from different devs).
+4. `hatch run start` to start the program (it can also be started from the **VS Code** debugger just by clicking **Run and Debug** while being on the `main.py` file for this project).
+
+#### Starting the Data Exchange Agent
+These steps will start the **Data Exchange Agent** locally. They must be performed after the corresponding resources have been set up in the Snowflake account (which is done when the **Data Migration Orchestrator** starts).
+1. Go to the `data-exchange-agent` directory (`cd data-exchange-agent`)
+2. Create a `configuration.toml` file in the `data-exchange-agent` (it will be ignored by **Git**). there is a `configuration_example.toml` file in that same directory, which can be used as an example. A minimal configuration is also shown below.
+3. `hatch run data-exchange-agent` to start the program (it can also be started from the **VS Code** debugger just by clicking **Run and Debug** while being on the `main.py` file for this project).
+
+Example minimal config:
+```toml
+selected_task_source = "snowflake_stored_procedure"
+
+[task_source.snowflake_stored_procedure]
+connection_name = <snowflake-connection-name>
+
+[application]
+workers = 1
+task_fetch_interval = 5s
+debug_mode = false
+affinity = <affinity>
+
+[connections.target.snowflake_connection_name]
+connection_name = <snowflake-connection-name>
+
+[connections.source.sqlserver]
+username = <sql-server-username>
+password = <sql-server-password>
+database = <sql-server-database-name>
+host = <sql-server-host>
+port = 1433
+```
+
+#### Creating a Workflow
+Prepare a `workflow-config.json` (see the [Workflow Configuration Reference](README.md#workflow-configuration-reference) in the README for the configuration format). The `affinity` field inside the config must match the affinity you are using for the **Data Exchange Agent**.
+
+```bash
+hatch run create-workflow workflow-config.json --connection-name <snowflake-connection-name>
+```
+
+This validates the configuration and creates a new record in the `SNOWCONVERT_AI.DATA_MIGRATION.WORKFLOW` table. If `--connection-name` is omitted, the connection from `SNOWFLAKE_CONNECTION_NAME` (or other env vars) is used.
+
+### Running all components in SPCS
+#### Starting the Data Migration Orchestrator
+You must run the `./data-migration-orchestrator/scripts/upload-orchestrator.sh` script (on the root of the repository). This will update the image for the **Data Exchange Agent** and push it to the image registry on the Snowflake account. This script takes an optional `--connection` parameter that receives the name of the Snowflake connection. If not provided, the default will be used.
+
+Then, execute this in the Snowflake account:
+```sql
+CREATE SERVICE IF NOT EXISTS SNOWCONVERT_AI.DATA_MIGRATION.DATA_MIGRATION_SERVICE
+IN COMPUTE POOL <compute-pool-name>
+FROM SPECIFICATION 
+$$
+spec:
+  containers:
+  - name: orchestrator
+    image: /SNOWCONVERT_AI/PUBLIC/IMAGES/data-migration-orchestrator:latest
+    env:
+      SNOWFLAKE_WAREHOUSE: <warehouse>
+    resources:
+      requests:
+        memory: 2Gi
+        cpu: 0.5
+$$
+QUERY_WAREHOUSE = <warehouse>;
+```
+
+#### Starting the Data Exchange Agent
+You must run the `./data-exchange-agent/scripts/upload-agent.sh` script (on the root of the repository). This will update the image for the **Data Exchange Agent** and push it to the image registry on the Snowflake account. This script takes an optional `--connection` parameter that receives the name of the Snowflake connection. If not provided, the default will be used.
+
+Then, execute this in the Snowflake account:
+```sql
+-- 1 Create secret and grant usage over it (Source System)
+CREATE OR REPLACE SECRET MY_DB.SECRETS.SQL_SERVER_PASSWORD
+  TYPE = PASSWORD
+  USERNAME = <sql-server-username>
+  PASSWORD = <sql-server-password>;
+
+GRANT USAGE ON SECRET MY_DB.SECRETS.SQL_SERVER_PASSWORD TO ROLE <data-migration-role-name>;
+
+-- 2. Create network rule
+CREATE OR REPLACE NETWORK RULE MY_DB.SECRETS.SQL_SERVER_PASSWORD
+  MODE = EGRESS
+  TYPE = HOST_PORT
+  VALUE_LIST = ('<sql-server-host>');
+
+-- 3. Create External Access Integration
+CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION SQL_SERVER_INTEGRATION
+  ALLOWED_NETWORK_RULES = (DM_DEMO_AZURE_SQL_EGRESS_RULE)
+  ENABLED = TRUE;
+
+GRANT USAGE ON INTEGRATION SQL_SERVER_INTEGRATION TO ROLE <data-migration-role-name>;
+
+-- 4. Create Service
+CREATE SERVICE SNOWCONVERT_AI.DATA_MIGRATION.DATA_EXCHANGE_SERVICE
+  IN COMPUTE POOL <compute-pool-name>
+  FROM SPECIFICATION $$
+    spec:
+      containers:
+        - name: agent
+          image: /snowconvert_ai/public/images/data-exchange-agent:latest
+          env:
+            # Source System Config
+            DATA_SOURCE_HOST: <sql-server-host>
+            DATA_SOURCE_PORT: <1433>
+            DATA_SOURCE_DATABASE: <sql-server-database>
+
+            # Snowflake Config
+            SNOWFLAKE_WAREHOUSE: <snowflake-warehouse>
+            
+            # Application
+            WORKER_COUNT: 1
+            AGENT_AFFINITY: <affinity>
+            
+          secrets:
+            # Secrets (Source System)
+          - snowflakeSecret: MY_DB.SECRETS.SQL_SERVER_PASSWORD
+            secretKeyRef: USERNAME
+            envVarName: DATA_SOURCE_USERNAME
+          - snowflakeSecret: MY_DB.SECRETS.SQL_SERVER_PASSWORD
+            secretKeyRef: PASSWORD
+            envVarName: DATA_SOURCE_PASSWORD
+  $$
+  -- This line links the network hole to the service
+  EXTERNAL_ACCESS_INTEGRATIONS = (SQL_SERVER_INTEGRATION)
+  MIN_INSTANCES = 6
+  MAX_INSTANCES = 6;
+```
+
+#### Creating a Workflow
+Creating a workflow is the same independently of where the components are deployed (see [Creating a Workflow](#creating-a-workflow-1) above).
+
+### Mixed approach
+You can combine the two approaches and run only the **Data Exchange Agent** locally and the **Data Migration Orchestrator** in SPCS (or the other way around). There is no significant difference when doing this.
+
+## Keeping Documentation up to Date
+When adding/modifying features, update the corresponding documentation:
+- **README.md** is the user-facing source of truth (architecture, usage, configuration reference).
+- **CONTRIBUTING.md** (this file) has developer content (commands, Docker, E2E, CI).
+- The [Data Exchange Agent README](../data-exchange-agent/README.md) is the source of truth for Worker Configuration.
+- Cursor rules and commands should be kept synchronized with the docs.
+
+When changing configuration models or behavior, update the README's configuration reference and the example workflow files in `docs/public/example-workflows/`.
+
+## Development Patterns
+### Task Model
+In the **Data Migration Orchestrator** we distribute work by converting workflows into sequences of many tasks:
+- Distributing the work in tasks help us achieve fault-tolerance, distribute the workload across several workers, and have "resumability".
+- These tasks can have successors/predecessors and be executed by different components of the architecture.
+- We strive for tasks to be idempotent, retryable, and moderately "quick" to execute. This might not always be achievable.
+- No specific executor is assigned to a task, although a specific executor type will be assigned depending on the kind of the task.
+- Each task has a payload that depends on the kind of the task. There are no guarantees on the time at which a task will be executed. Because of that, we can't assume that a sucessor task will be executed immediately after its predecessor (hours/days could have passed since).
+
+When adding features to the Orchestrator, consider this task model. For more information, you can visit the `task-model.md` document.
+
+### Creating/Queuing Tasks
+Create tasks using the `TaskBuilder` class and then queue them using the `TaskQueue`. When creating sequences of dependent tasks, you can use the `push_task_chain` method of the `TaskQueue`.
+
+### Manipulating Stage Paths
+We constantly are putting data/metadata into files in different paths of stages and then reading that data/metadata (or copying it into tables). For example, the default internal stage is `SNOWCONVERT_AI.DATA_MIGRATION.TASK_RESULTS` and we normally upload metadata/data of the tables we're moving.
+
+We try to avoid passing stage paths through task payloads. Instead, we try to have a standardize pattern for stage paths. For example, if you want to access the path at which the schema information for a table was uploaded, then it might look something like this: `<stage-id>/workflows/<workflow-id>/table/<table-metadata-id>/schema/`.
+
+### Scopes
+Scopes are described in detail in the `scopes.md` doc. Those are useful for understanding what is the purpose of each task and to what "domain" it applies.
+
+When creating scopes, we should use the `ScopeBuilder`. There are other helper functions that can be used when creating scopes (specially if the scope is very commonly created).
+
+### Schema Migrations
+When changing the schema (tables in the `SNOWCONVERT_AI.DATA_MIGRATION` schema, for example), we should do it through schema migrations. Check the `schema-migrations.md` doc for more information about this topic.
+
+## Anti-patterns
+1. We want to avoid having duplicated dataclasses (or dataclasses that are almost equivalent) without a good reason.
+2. We do not want to rely too much on magic strings. In cases where we are dealing with categories/kinds, we should prefer to create a `StrEnum`.
+3. We want to avoid "carrying" too much information across tasks, particularly if you are sending a piece of information that is not meant for a task, but rather for a future task that might be created by the immediate task you are creating. We should strive to read configuration values from the TABLE_METADATA configuration column, for example. This also applies to the PARTITION_METADATA synchronization data, for example.
+4. We want to avoid using `Any` if possible. In general, we try to keep the code as type-safe as possible.
+5. We want to avoid complex types without a meaningful name. For example, types like `dict[str, dict[str, str]]` might be complex to understand when reading the code. A type alias or a dedicated class can help here.
+6. Let's not make non-deterministic tests, or unit tests that can take a lot of time. Running all tests should be matter of less than 10 seconds.
+
+## Considerations
+1. If the **Workflow Config** schema changes, update the `example-workflow-config.json` file. Also, update the mechanism for schema validation in **SnowConvert Desktop**.
+2. If the **Data Exchange Agent Config** schema changes, update the `data-exchange-agent/src/data_exchange_agent/configuration_example.toml` (and possibly the `data-exchange-agent/docker-artifacts/configuration.template.toml`).
+3. For development, it is useful to set orchestrator affinity (by setting the `DATA_MIGRATION_ORCHESTRATOR_AFFINITY` to your affinity value). This will prevent other orchestrator instances (from other devs) from picking up the workflows you have created (and the other way around).
+
+## How to add a new platform?
+### Basic Integration
+For integrating a new platform, it suffices to:
+1. Add the new value to the `SourceType` enum (to `shared/enums/source_type.py`). 
+2. Add the corresponding *default mappings* in `default_mappings.py`. Some examples: `sqlserver_default_mappings.py` and `redshift_default_mappings.py`.
+3. Create the directory for this new platform in the `src/data_migration_orchestrator/data_migration_cloud/platforms` directory. Follow the examples of existing platforms.
+4. Add the corresponding subclass of `BaseQueryBuilder` for the platform. Some examples: `RedshiftQueryBuilder` and `SqlServerQueryBuilder`. Make sure to implement all methods and override base methods if needed.
+5. Add the corresponding subclass of `BasePlatform` for the platform. Some examples: `RedshiftPlatform` and `SqlServerPlatform`. Make sure to implement all methods and override base methods if needed.
+6. In `src/data_migration_orchestrator/data_migration_cloud/platforms/__init__.py`, you must register the platform and corresponding aliases (if any).
+
+Some changes will likely be needed on the `data-exchange-agent` project.
+Finally make sure to do appropriate manual testing and include new unit tests. 
+
+### Optimizations
+Some platforms might have an "optimized" way of extracting the data and pushing it to a Snowflake stage (internal or external).
+
+#### Examples
+##### SQL Server**
+**BCP** tool is highly performant and can be used to extract the data to CSV files. This approach requires changes only on the **Data Exchange Agent** (and **BCP** is only leveraged for "bulk" data movement operations, not for small metadata queries).
+
+##### Redshift
+It is common to use the **UNLOAD** strategy, in which an **UNLOAD** statement is issued to the **Redshift** insteance (with the appropriate SQL query) and that creates a Parquet file in a **S3** bucket. On the side of the **Data Migration Orchestrator**, the workflow must receive configuration to use the **UNLOAD** strategy (this must also include the name of the external stage that is pointing to the **S3** bucket). This approach requires changes on both the **Data Exchange Agent** (to understand how to issue the **UNLOAD** statement) and in the **Data Migration Orchestrator** (to read the configuration and understand where to read the data from).
+
+#### Considerations
+- Take into account if a new extraction strategy must be added to the **Data Migration Orchestrator**. This would require changes in the workflow configuration models.
+- Take into account if the optimization only requires changes to the **Data Exchange Agent** (which is preferred). An example of this is the **BCP** approach for **SQL Server**. 

snowflake_data_migration_orchestrator-0.2.0/Dockerfile

@@ -0,0 +1,30 @@
+# Use Python 3.11 slim image
+FROM python:3.11-slim
+
+# Set working directory
+WORKDIR /app
+
+# Copy shared directory from parent context (required by pyproject.toml force-include)
+COPY shared /shared
+
+# Copy project files
+COPY data-migration-orchestrator/pyproject.toml data-migration-orchestrator/README.md ./
+COPY data-migration-orchestrator/src/ ./src/
+
+# Install the package and dependencies
+RUN pip install --no-cache-dir -e .
+# TODO(SNOW-2860314): move dependency installation before the part in which we copy the source code to leverage Docker layer caching
+
+# Set the Python path
+ENV PYTHONPATH=/app/src
+
+# Disable Python output buffering to see logs in real-time
+ENV PYTHONUNBUFFERED=1
+
+# Create non-root user and pre-create writable dirs
+RUN groupadd --system appgroup && useradd --system --gid appgroup appuser \
+    && mkdir -p /app/logs && chown appuser:appgroup /app/logs
+USER appuser
+
+# Run the main module
+CMD ["python", "-u", "-m", "data_migration_orchestrator"]