dq-export-connector 0.1.0.dev1__tar.gz

dq_export_connector-0.1.0.dev1/.gitignore

@@ -0,0 +1,63 @@
+.DS_Store
+**/build/
+.idea/
+.vscode/*
+!.vscode/tasks.json
+target/
+node_modules/
+result
+/result-*
+**/*.o
+env
+venv
+.venv
+.direnv
+**/perf.data*
+infra/packages/hadoop/binary.tar.gz
+.terraform
+config.tf.json
+*.tfstate.*
+*.tfstate
+**/.deploy-gc
+.vagrant
+.pnpm-debug.log
+.yarn
+__pycache__
+trusted/dcr-compiler/ddc/src/data_lab/**/*.png
+trusted/dcr-compiler/ddc/src/data_lab/**/*.json
+e2e-tests/frontend/confidence-check-config.json
+.terraform.lock.hcl
+dmp/analysis-scripts/results/*
+.env
+!tools/dqdev/internal/target/
+docusaurus/style-guide.md
+**/.claude/settings.local.json
+**/.claude/agent-memory-local/
+.claude/*.lock
+.claude/worktrees/
+**/CLAUDE.local.md
+**/*.parquet/**
+.pnpm-store/
+.turbo/
+
+# Agentic-coding workflow (local, not part of this repo)
+.workflow/
+.workflow-backup-*
+
+# Local-only Google Ads/DV360 test scripts (kept out of the PR)
+dmp/dqdmp/scripts/
+
+# Playwright MCP screenshots (local only)
+.playwright-mcp/
+
+# Claude Design sync build artifacts (local only).
+**/.design-sync/sb-reference/
+**/.design-sync/learnings/
+**/.design-sync/.cache/
+**/.design-sync/node_modules
+.ds-sync/
+ds-bundle/
+
+# DMP creation-prefill tests (local only)
+avato-backend/frontend/decentriq-platform/src/features/dmp/models/creationPrefill.test.ts
+avato-backend/frontend/decentriq-platform/src/features/dmp/utils/creationPrefill/*.test.ts

dq_export_connector-0.1.0.dev1/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.5
+Name: dq-export-connector
+Version: 0.1.0.dev1
+Summary: Worker logic for the Export Connector DCR.
+Author: Decentriq
+Requires-Python: >=3.10
+Requires-Dist: obstore<0.12,>=0.11.1
+Requires-Dist: pydantic<3,>=2.7
+Description-Content-Type: text/markdown
+
+# dq-export-connector
+
+Worker logic for the **Export Connector DCR**: one package for every
+Connector Kind, each in its own sub-package. The **bucket** kind uploads each
+Input's dataset to S3, Azure Blob or GCS through
+[`obstore`](https://developmentseed.org/obstore/).
+
+ddc pins this package by hash in a fully locked `requirements.txt` and runs
+`dq_export_connector.runner.main` in the task's Python environment. It is
+published to PyPI, since the compiled graph installs it from there.
+
+- `dq_export_connector.runner`: the entry point. Reads `/input/config.json`
+  and `/input/credentials.json`, sets up logging, and dispatches on the
+  Connector Kind and the action.
+- `dq_export_connector.redact`: keeps every value from the sealed half, and
+  URL query strings, out of log lines and result messages.
+- `dq_export_connector.bucket`: the bucket kind (`ValidateConnection` and
+  `Export`).
+- `dq_export_connector._models`: pydantic models generated from ddc's JSON
+  schemas, so the config the worker reads and the receipt and validation
+  result it writes match ddc's Rust types. Regenerate them with
+  `scripts/codegen-models.sh` after the types change and
+  `decentriq_dcr_compiler` has been rebuilt; a test fails while they are
+  stale.
+
+```sh
+uv run pytest            # unit tests, against obstore's in-memory store
+uv run pytest -m live    # against real buckets, credentials from the environment
+uv run mypy dq_export_connector tests
+```
+
+## Releasing
+
+Bump `version` in `pyproject.toml`, then push a `dq-export-connector-v<version>`
+tag; the `Publish dq-export-connector to PyPI` workflow tests, builds and uploads
+it. Use `.devN` versions while developing; PyPI releases can't be replaced.
+ddc then pins the new version in a new `export_connector/v0/worker/<version>/`.

dq_export_connector-0.1.0.dev1/README.md

@@ -0,0 +1,37 @@
+# dq-export-connector
+
+Worker logic for the **Export Connector DCR**: one package for every
+Connector Kind, each in its own sub-package. The **bucket** kind uploads each
+Input's dataset to S3, Azure Blob or GCS through
+[`obstore`](https://developmentseed.org/obstore/).
+
+ddc pins this package by hash in a fully locked `requirements.txt` and runs
+`dq_export_connector.runner.main` in the task's Python environment. It is
+published to PyPI, since the compiled graph installs it from there.
+
+- `dq_export_connector.runner`: the entry point. Reads `/input/config.json`
+  and `/input/credentials.json`, sets up logging, and dispatches on the
+  Connector Kind and the action.
+- `dq_export_connector.redact`: keeps every value from the sealed half, and
+  URL query strings, out of log lines and result messages.
+- `dq_export_connector.bucket`: the bucket kind (`ValidateConnection` and
+  `Export`).
+- `dq_export_connector._models`: pydantic models generated from ddc's JSON
+  schemas, so the config the worker reads and the receipt and validation
+  result it writes match ddc's Rust types. Regenerate them with
+  `scripts/codegen-models.sh` after the types change and
+  `decentriq_dcr_compiler` has been rebuilt; a test fails while they are
+  stale.
+
+```sh
+uv run pytest            # unit tests, against obstore's in-memory store
+uv run pytest -m live    # against real buckets, credentials from the environment
+uv run mypy dq_export_connector tests
+```
+
+## Releasing
+
+Bump `version` in `pyproject.toml`, then push a `dq-export-connector-v<version>`
+tag; the `Publish dq-export-connector to PyPI` workflow tests, builds and uploads
+it. Use `.devN` versions while developing; PyPI releases can't be replaced.
+ddc then pins the new version in a new `export_connector/v0/worker/<version>/`.

dq_export_connector-0.1.0.dev1/dq_export_connector/_models/bucket_validation_result.py

@@ -0,0 +1,63 @@
+# generated by datamodel-codegen:
+#   filename:  bucket_validation_result.json
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, RootModel
+
+
+class BucketValidationResult1(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    status: Literal['OK']
+
+
+class ExportMessage(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    code: Annotated[
+        str,
+        Field(
+            description='SCREAMING_SNAKE_CASE, such as `ACCESS_DENIED`. A string rather than an\nenum, so a worker release can add codes that readers built against an\nolder ddc still accept.'
+        ),
+    ]
+    message: Annotated[
+        str,
+        Field(
+            description='Safe to show to the user: no credentials and no dataset values.'
+        ),
+    ]
+
+
+class BucketValidationResult2(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    status: Literal['OK_WITH_WARNINGS']
+    warnings: list[ExportMessage]
+
+
+class BucketValidationResult3(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    errors: list[ExportMessage]
+    status: Literal['ERROR']
+
+
+class BucketValidationResult(
+    RootModel[
+        BucketValidationResult1 | BucketValidationResult2 | BucketValidationResult3
+    ]
+):
+    root: Annotated[
+        BucketValidationResult1 | BucketValidationResult2 | BucketValidationResult3,
+        Field(
+            description='The result of a Bucket `ValidateConnection`. Connection problems are\nreported here, and the task still succeeds.',
+            title='BucketValidationResult',
+        ),
+    ]

dq_export_connector-0.1.0.dev1/dq_export_connector/_models/export_connector_credentials.py

@@ -0,0 +1,84 @@
+# generated by datamodel-codegen:
+#   filename:  export_connector_credentials.json
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from pydantic import BaseModel, ConfigDict, Field, RootModel
+
+
+class S3(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    access_key: Annotated[str, Field(alias='accessKey')]
+    secret_key: Annotated[str, Field(alias='secretKey')]
+
+
+class BucketCredentials1(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+        populate_by_name=True,
+    )
+    s3: S3
+
+
+class AzureBlob(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    sas_token: Annotated[
+        str,
+        Field(
+            alias='sasToken',
+            description='A container SAS token — the scoped, expiring grant Azure recommends\nover an account key. Only the grant: the account and container it\napplies to are pinned by the signed `BucketConnection`, so rotating\nthe token can never repoint the connector.',
+        ),
+    ]
+
+
+class BucketCredentials2(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+        populate_by_name=True,
+    )
+    azure_blob: Annotated[AzureBlob, Field(alias='azureBlob')]
+
+
+class Gcs(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    service_account_json: Annotated[str, Field(alias='serviceAccountJson')]
+
+
+class BucketCredentials3(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+        populate_by_name=True,
+    )
+    gcs: Gcs
+
+
+class BucketCredentials(
+    RootModel[BucketCredentials1 | BucketCredentials2 | BucketCredentials3]
+):
+    root: BucketCredentials1 | BucketCredentials2 | BucketCredentials3
+
+
+class ExportConnectorCredentials1(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+        populate_by_name=True,
+    )
+    bucket: BucketCredentials
+
+
+class ExportConnectorCredentials(RootModel[ExportConnectorCredentials1]):
+    root: Annotated[
+        ExportConnectorCredentials1,
+        Field(
+            description='Keyed by Connector Kind, in step with [`ExportConnectorKind`]. The worker\nreads it at `/input/credentials.json`.',
+            title='ExportConnectorCredentials',
+        ),
+    ]

dq_export_connector-0.1.0.dev1/dq_export_connector/_models/export_receipt.py

@@ -0,0 +1,152 @@
+# generated by datamodel-codegen:
+#   filename:  export_receipt.json
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, RootModel
+
+
+class ExportMessage(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    code: Annotated[
+        str,
+        Field(
+            description='SCREAMING_SNAKE_CASE, such as `ACCESS_DENIED`. A string rather than an\nenum, so a worker release can add codes that readers built against an\nolder ddc still accept.'
+        ),
+    ]
+    message: Annotated[
+        str,
+        Field(
+            description='Safe to show to the user: no credentials and no dataset values.'
+        ),
+    ]
+
+
+class ExportTotals(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    bytes: Annotated[int, Field(ge=0)]
+    files: Annotated[int, Field(ge=0)]
+    rows: Annotated[
+        int | None,
+        Field(
+            description='`None` unless the worker counted the rows of every file.', ge=0
+        ),
+    ] = None
+
+
+class ExportedFile(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    bytes: Annotated[int, Field(ge=0)]
+    path: Annotated[str, Field(description='The object key in the bucket.')]
+    rows: Annotated[
+        int | None, Field(description="`None` where the worker can't count rows.", ge=0)
+    ] = None
+
+
+class InputStatus(Enum):
+    success = 'SUCCESS'
+    failed = 'FAILED'
+
+
+class BucketInputReceipt(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    errors: list[ExportMessage]
+    files: list[ExportedFile]
+    leaf_id: Annotated[str, Field(alias='leafId')]
+    status: InputStatus
+    totals: ExportTotals
+    warnings: list[ExportMessage]
+
+
+class ExportReceiptV01(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    errors: Annotated[
+        list[ExportMessage],
+        Field(
+            description='Problems tied to no single Input, such as credentials rejected before\nany upload.'
+        ),
+    ]
+    export_prefix: Annotated[
+        str | None,
+        Field(
+            alias='exportPrefix',
+            description='The export prefix with its date tokens filled; `None` without one.',
+        ),
+    ] = None
+    inputs: list[BucketInputReceipt]
+    totals: ExportTotals
+    warnings: list[ExportMessage]
+    type: Literal['BUCKET']
+
+
+class ExportReceiptV0(RootModel[ExportReceiptV01]):
+    root: ExportReceiptV01
+
+
+class ExportReceipt1(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    version: Literal['V0']
+    errors: Annotated[
+        list[ExportMessage],
+        Field(
+            description='Problems tied to no single Input, such as credentials rejected before\nany upload.'
+        ),
+    ]
+    export_prefix: Annotated[
+        str | None,
+        Field(
+            alias='exportPrefix',
+            description='The export prefix with its date tokens filled; `None` without one.',
+        ),
+    ] = None
+    inputs: list[BucketInputReceipt]
+    totals: ExportTotals
+    warnings: list[ExportMessage]
+    type: Literal['BUCKET']
+
+
+class ExportReceipt(RootModel[ExportReceipt1]):
+    root: Annotated[
+        ExportReceipt1,
+        Field(
+            description='A released version never changes: a change is a new version, and readers\nkeep reading every version ever written.',
+            title='ExportReceipt',
+        ),
+    ]
+
+
+class BucketExportReceiptV0(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    errors: Annotated[
+        list[ExportMessage],
+        Field(
+            description='Problems tied to no single Input, such as credentials rejected before\nany upload.'
+        ),
+    ]
+    export_prefix: Annotated[
+        str | None,
+        Field(
+            alias='exportPrefix',
+            description='The export prefix with its date tokens filled; `None` without one.',
+        ),
+    ] = None
+    inputs: list[BucketInputReceipt]
+    totals: ExportTotals
+    warnings: list[ExportMessage]

dq_export_connector-0.1.0.dev1/dq_export_connector/_models/export_worker_config.py

@@ -0,0 +1,183 @@
+# generated by datamodel-codegen:
+#   filename:  export_worker_config.json
+
+from __future__ import annotations
+
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, RootModel
+
+
+class BucketConnection1(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    bucket: str
+    path_prefix: Annotated[str | None, Field(alias='pathPrefix')] = None
+    region: str
+    type: Literal['s3']
+
+
+class BucketConnection2(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    account_name: Annotated[str, Field(alias='accountName')]
+    container: str
+    path_prefix: Annotated[str | None, Field(alias='pathPrefix')] = None
+    type: Literal['azureBlob']
+
+
+class BucketConnection3(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    bucket: str
+    path_prefix: Annotated[str | None, Field(alias='pathPrefix')] = None
+    type: Literal['gcs']
+
+
+class BucketConnection(
+    RootModel[BucketConnection1 | BucketConnection2 | BucketConnection3]
+):
+    root: Annotated[
+        BucketConnection1 | BucketConnection2 | BucketConnection3,
+        Field(
+            description='Cloud-bucket addressing (no credentials). Provider-structurally paired with\nthe matching [`BucketCredentials`] variant in the confidential half.'
+        ),
+    ]
+
+
+class BucketInput(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    destination_directory: Annotated[str, Field(alias='destinationDirectory')]
+    leaf_id: Annotated[str, Field(alias='leafId')]
+
+
+class ExportConnectorKind1(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    connection: BucketConnection
+    export_prefix: Annotated[
+        str | None,
+        Field(
+            alias='exportPrefix',
+            description='May hold `{yyyy}`, `{mm}`, `{dd}` and `{hh}`, which the worker fills in\nUTC when a run starts.',
+        ),
+    ] = None
+    inputs: list[BucketInput]
+    type: Literal['bucket']
+
+
+class ExportConnectorKind(RootModel[ExportConnectorKind1]):
+    root: ExportConnectorKind1
+
+
+class ExportParams1(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    type: Literal['bucket']
+
+
+class ExportParams(RootModel[ExportParams1]):
+    root: ExportParams1
+
+
+class ProvisionedPolicy(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    leaf_id: Annotated[str, Field(alias='leafId')]
+    policy_id: Annotated[str, Field(alias='policyId')]
+    provisioned_at: Annotated[
+        str, Field(alias='provisionedAt', description='RFC 3339.')
+    ]
+    provisioned_by_email: Annotated[
+        str,
+        Field(
+            alias='provisionedByEmail',
+            description='Whose READ on the dataset every run re-checks.',
+        ),
+    ]
+
+
+class ValidateConnectionParams1(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    type: Literal['bucket']
+
+
+class ValidateConnectionParams(RootModel[ValidateConnectionParams1]):
+    root: ValidateConnectionParams1
+
+
+class BucketExportConfig(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    connection: BucketConnection
+    export_prefix: Annotated[
+        str | None,
+        Field(
+            alias='exportPrefix',
+            description='May hold `{yyyy}`, `{mm}`, `{dd}` and `{hh}`, which the worker fills in\nUTC when a run starts.',
+        ),
+    ] = None
+    inputs: list[BucketInput]
+
+
+class ValidateConnection(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    params: ValidateConnectionParams
+
+
+class ExportConnectorDcrComputeAction1(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+        populate_by_name=True,
+    )
+    validate_connection: Annotated[
+        ValidateConnection, Field(alias='validateConnection')
+    ]
+
+
+class Export(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    params: ExportParams
+
+
+class ExportConnectorDcrComputeAction2(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+        populate_by_name=True,
+    )
+    export: Export
+
+
+class ExportConnectorDcrComputeAction(
+    RootModel[ExportConnectorDcrComputeAction1 | ExportConnectorDcrComputeAction2]
+):
+    root: Annotated[
+        ExportConnectorDcrComputeAction1 | ExportConnectorDcrComputeAction2,
+        Field(
+            description='Compute actions for an Export Connector DCR, started through `POST /jobs`.'
+        ),
+    ]
+
+
+class ExportWorkerConfig(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    action: ExportConnectorDcrComputeAction
+    connector: ExportConnectorKind
+    policies: list[ProvisionedPolicy]