topos-node 0.1.0__py3-none-any.whl

topos/services/local.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from ..core.api_models import (
+    DeviceInfoResponse,
+    DeviceNameResponse,
+    PairDeviceResponse,
+    PairingCodeResponse,
+    StoreMessageResponse,
+    SyncDatabaseResponse,
+    SyncResponse,
+)
+from ..core import state
+from ..config.settings import settings
+from fastapi import HTTPException, status
+
+
+class LocalDbService:
+    async def store_message(self, payload: Dict[str, Any]) -> StoreMessageResponse:
+        _ = payload
+        raise NotImplementedError("LocalDbService not implemented yet")
+
+    async def get_oplog(self, dataset_id: Optional[str], limit: int, offset: int) -> Dict[str, Any]:
+        _ = (dataset_id, limit, offset)
+        raise NotImplementedError("LocalDbService not implemented yet")
+
+    async def get_messages(self, dataset_id: Optional[str], limit: int, offset: int) -> Dict[str, Any]:
+        _ = (dataset_id, limit, offset)
+        raise NotImplementedError("LocalDbService not implemented yet")
+
+    async def replay_projection(self, dataset_id: Optional[str]) -> Dict[str, Any]:
+        _ = dataset_id
+        raise NotImplementedError("LocalDbService not implemented yet")
+
+    async def reset_database(self) -> Dict[str, Any]:
+        raise NotImplementedError("LocalDbService not implemented yet")
+
+    async def sync_database(self) -> SyncDatabaseResponse:
+        raise NotImplementedError("LocalDbService not implemented yet")
+
+    async def backup_database(self, encrypted: bool) -> Any:
+        _ = encrypted
+        raise NotImplementedError("LocalDbService not implemented yet")
+
+    async def restore_database(self, file, authenticated_user_id: str, encrypted: bool) -> Dict[str, Any]:
+        _ = (file, authenticated_user_id, encrypted)
+        raise NotImplementedError("LocalDbService not implemented yet")
+
+
+class LocalSyncService:
+    async def trigger_sync(self) -> SyncResponse:
+        raise NotImplementedError("LocalSyncService not implemented yet")
+
+
+class LocalDeviceService:
+    async def get_pairing_code(self) -> PairingCodeResponse:
+        raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Pairing not implemented")
+
+    async def pair_device(self, pairing_code: str, keep_existing_data: bool) -> PairDeviceResponse:
+        _ = (pairing_code, keep_existing_data)
+        raise HTTPException(status_code=status.HTTP_501_NOT_IMPLEMENTED, detail="Pairing not implemented")
+
+    async def get_device_info(self) -> DeviceInfoResponse:
+        # Get user_id from database (set by connection_info handler) or fall back to settings
+        user_id = None
+        if state.db_conn:
+            from ..core.state import get_user_id
+            user_id = get_user_id(state.db_conn)
+        if not user_id:
+            user_id = settings.topos_user_id
+        dataset_id = f"{user_id}:{settings.topos_default_dataset_id}" if user_id else None
+        sync_connected = state.sync_client.is_connected() if state.sync_client else False
+        sync_enabled = settings.enable_sync and settings.get_sync_url() is not None
+
+        last_sync_at = None
+        last_received_hlc_ts = None
+        last_received_op_id = None
+        if state.db_conn:
+            last_sync_at = state.get_engine_config_value(state.db_conn, "last_sync_at")
+            last_received_hlc_ts = state.get_engine_config_value(state.db_conn, "last_received_hlc_ts")
+            last_received_op_id = state.get_engine_config_value(state.db_conn, "last_received_op_id")
+
+        device_name = settings.engine_name or state.get_system_info().get("hostname")
+
+        return DeviceInfoResponse(
+            user_id=user_id,
+            dataset_id=dataset_id,
+            sync_connected=sync_connected,
+            sync_enabled=sync_enabled,
+            engine_class=state.get_engine_class(),
+            engine_mode=state.get_engine_mode(),
+            llm_enabled=settings.enable_llm and state.get_engine_mode() == "full",
+            database_mode=settings.topos_database_mode,
+            engine_name=device_name,
+            engine_version=None,
+            system=state.get_system_info(),
+            last_sync_at=last_sync_at,
+            last_received_hlc_ts=last_received_hlc_ts,
+            last_received_op_id=last_received_op_id,
+            oplog_count=None,
+            oplog_bytes=None,
+            ops_since_last_sync=None,
+            oplog_bytes_since_last_sync=None,
+        )
+
+    async def set_device_name(self, device_name: str) -> DeviceNameResponse:
+        name = device_name.strip()
+        if not name:
+            raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Device name cannot be empty")
+        if len(name) > 64:
+            raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Device name cannot exceed 64 characters")
+
+        if state.db_conn:
+            state.set_engine_config_value(state.db_conn, "device_name", name)
+
+        return DeviceNameResponse(status="ok", device_name=name)
+
+
+class LocalLLMService:
+    async def generate(self, payload: Dict[str, Any]) -> Dict[str, Any]:
+        _ = payload
+        raise NotImplementedError("LocalLLMService not implemented yet")

topos/services/postgres.py

@@ -0,0 +1,385 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+from uuid import uuid4
+
+from fastapi import HTTPException, status
+
+from ..config.settings import settings
+from ..core.api_models import (
+    DeviceInfoResponse,
+    DeviceNameResponse,
+    PairDeviceResponse,
+    PairingCodeResponse,
+    StoreMessageResponse,
+    SyncDatabaseResponse,
+    SyncResponse,
+)
+from ..storage.db.postgres import (
+    PostgresConfigurationError,
+    connect_postgres,
+    execute_query,
+    fetch_all,
+)
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _http_error(status_code: int, code: str, message: str) -> HTTPException:
+    return HTTPException(status_code=status_code, detail={"code": code, "message": message})
+
+
+def _tenant_from_dataset(dataset_id: Optional[str]) -> str:
+    ds = (dataset_id or "").strip()
+    if not ds:
+        raise _http_error(status.HTTP_400_BAD_REQUEST, "dataset_required", "dataset_id is required")
+    if ":" in ds:
+        tenant, _ = ds.split(":", 1)
+        tenant = tenant.strip()
+    else:
+        tenant = ds
+    if not tenant:
+        raise _http_error(status.HTTP_400_BAD_REQUEST, "tenant_required", "dataset_id must include tenant scope")
+    return tenant
+
+
+def _assert_authenticated_tenant(tenant_id: str) -> None:
+    expected = (settings.topos_user_id or "").strip()
+    if not expected:
+        return
+    if tenant_id != expected:
+        raise _http_error(
+            status.HTTP_403_FORBIDDEN,
+            "tenant_access_denied",
+            "Requested dataset tenant does not match authenticated tenant",
+        )
+
+
+class PostgresDbService:
+    async def store_message(self, payload: Dict[str, Any]) -> StoreMessageResponse:
+        dataset_id = (payload.get("dataset_id") or "").strip()
+        tenant_id = _tenant_from_dataset(dataset_id)
+        _assert_authenticated_tenant(tenant_id)
+
+        sender_type = (payload.get("sender_type") or "").strip()
+        if not sender_type:
+            raise _http_error(status.HTTP_400_BAD_REQUEST, "sender_type_required", "sender_type is required")
+        content = (payload.get("content") or "").strip()
+        if not content:
+            raise _http_error(status.HTTP_400_BAD_REQUEST, "content_required", "content is required")
+
+        message_id = (payload.get("message_id") or "").strip() or str(uuid4())
+        op_id = str(uuid4())
+        ts = (payload.get("ts") or "").strip() or _utc_now_iso()
+        user_id = (payload.get("user_id") or "").strip() or None
+
+        try:
+            with connect_postgres() as conn:
+                execute_query(
+                    conn,
+                    """
+                    INSERT INTO messages (tenant_id, dataset_id, message_id, sender_type, content, ts, user_id)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s)
+                    """,
+                    (tenant_id, dataset_id, message_id, sender_type, content, ts, user_id),
+                )
+                execute_query(
+                    conn,
+                    """
+                    INSERT INTO oplog (tenant_id, dataset_id, op_id, op_type, payload_json, hlc_ts)
+                    VALUES (%s, %s, %s, %s, %s, %s)
+                    """,
+                    (
+                        tenant_id,
+                        dataset_id,
+                        op_id,
+                        "store_message",
+                        json.dumps(
+                            {
+                                "message_id": message_id,
+                                "sender_type": sender_type,
+                                "content": content,
+                                "ts": ts,
+                            }
+                        ),
+                        ts,
+                    ),
+                )
+        except HTTPException:
+            raise
+        except PostgresConfigurationError as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_not_configured", str(exc)) from exc
+        except Exception as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_write_failed", str(exc)) from exc
+
+        return StoreMessageResponse(op_id=op_id, message_id=message_id, status="ok")
+
+    async def get_oplog(self, dataset_id: Optional[str], limit: int, offset: int) -> Dict[str, Any]:
+        tenant_id = _tenant_from_dataset(dataset_id)
+        _assert_authenticated_tenant(tenant_id)
+        page_limit = max(1, min(int(limit), 1000))
+        page_offset = max(0, int(offset))
+        try:
+            with connect_postgres() as conn:
+                rows = fetch_all(
+                    conn,
+                    """
+                    SELECT op_id, op_type, payload_json, hlc_ts, dataset_id
+                    FROM oplog
+                    WHERE tenant_id = %s AND dataset_id = %s
+                    ORDER BY hlc_ts DESC
+                    LIMIT %s OFFSET %s
+                    """,
+                    (tenant_id, dataset_id, page_limit, page_offset),
+                )
+        except HTTPException:
+            raise
+        except PostgresConfigurationError as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_not_configured", str(exc)) from exc
+        except Exception as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_read_failed", str(exc)) from exc
+
+        items = []
+        for row in rows:
+            items.append(
+                {
+                    "op_id": row[0],
+                    "op_type": row[1],
+                    "payload": json.loads(row[2]) if row[2] else {},
+                    "hlc_ts": row[3],
+                    "dataset_id": row[4],
+                }
+            )
+        return {"status": "ok", "dataset_id": dataset_id, "items": items, "limit": page_limit, "offset": page_offset}
+
+    async def get_messages(self, dataset_id: Optional[str], limit: int, offset: int) -> Dict[str, Any]:
+        tenant_id = _tenant_from_dataset(dataset_id)
+        _assert_authenticated_tenant(tenant_id)
+        page_limit = max(1, min(int(limit), 1000))
+        page_offset = max(0, int(offset))
+        try:
+            with connect_postgres() as conn:
+                rows = fetch_all(
+                    conn,
+                    """
+                    SELECT message_id, sender_type, content, ts, user_id, dataset_id
+                    FROM messages
+                    WHERE tenant_id = %s AND dataset_id = %s
+                    ORDER BY ts DESC
+                    LIMIT %s OFFSET %s
+                    """,
+                    (tenant_id, dataset_id, page_limit, page_offset),
+                )
+        except HTTPException:
+            raise
+        except PostgresConfigurationError as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_not_configured", str(exc)) from exc
+        except Exception as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_read_failed", str(exc)) from exc
+
+        items = []
+        for row in rows:
+            items.append(
+                {
+                    "message_id": row[0],
+                    "sender_type": row[1],
+                    "content": row[2],
+                    "ts": row[3],
+                    "user_id": row[4],
+                    "dataset_id": row[5],
+                }
+            )
+        return {"status": "ok", "dataset_id": dataset_id, "messages": items, "limit": page_limit, "offset": page_offset}
+
+    async def replay_projection(self, dataset_id: Optional[str]) -> Dict[str, Any]:
+        tenant_id = _tenant_from_dataset(dataset_id)
+        _assert_authenticated_tenant(tenant_id)
+        try:
+            with connect_postgres() as conn:
+                rows = fetch_all(
+                    conn,
+                    """
+                    SELECT COUNT(*) FROM messages
+                    WHERE tenant_id = %s AND dataset_id = %s
+                    """,
+                    (tenant_id, dataset_id),
+                )
+        except HTTPException:
+            raise
+        except PostgresConfigurationError as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_not_configured", str(exc)) from exc
+        except Exception as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_replay_failed", str(exc)) from exc
+        replayed = int(rows[0][0]) if rows else 0
+        return {"status": "ok", "dataset_id": dataset_id, "replayed_messages": replayed}
+
+    async def reset_database(self) -> Dict[str, Any]:
+        raise _http_error(
+            status.HTTP_403_FORBIDDEN,
+            "reset_forbidden",
+            "Hosted mode database reset is not allowed without tenant-scoped maintenance flow",
+        )
+
+    async def sync_database(self) -> SyncDatabaseResponse:
+        return SyncDatabaseResponse(
+            status="ok",
+            message="Hosted mode uses server-backed Postgres and does not require local sync export",
+        )
+
+    async def backup_database(self, encrypted: bool) -> Any:
+        _ = encrypted
+        try:
+            with connect_postgres() as conn:
+                messages = fetch_all(
+                    conn,
+                    """
+                    SELECT tenant_id, dataset_id, message_id, sender_type, content, ts, user_id
+                    FROM messages
+                    ORDER BY ts ASC
+                    """,
+                )
+                oplog = fetch_all(
+                    conn,
+                    """
+                    SELECT tenant_id, dataset_id, op_id, op_type, payload_json, hlc_ts
+                    FROM oplog
+                    ORDER BY hlc_ts ASC
+                    """,
+                )
+        except PostgresConfigurationError as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_not_configured", str(exc)) from exc
+        except Exception as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_backup_failed", str(exc)) from exc
+
+        payload = {
+            "messages": [
+                {
+                    "tenant_id": row[0],
+                    "dataset_id": row[1],
+                    "message_id": row[2],
+                    "sender_type": row[3],
+                    "content": row[4],
+                    "ts": row[5],
+                    "user_id": row[6],
+                }
+                for row in messages
+            ],
+            "oplog": [
+                {
+                    "tenant_id": row[0],
+                    "dataset_id": row[1],
+                    "op_id": row[2],
+                    "op_type": row[3],
+                    "payload_json": row[4],
+                    "hlc_ts": row[5],
+                }
+                for row in oplog
+            ],
+        }
+        return json.dumps(payload).encode("utf-8")
+
+    async def restore_database(self, file, authenticated_user_id: str, encrypted: bool) -> Dict[str, Any]:
+        _ = encrypted
+        tenant_id = (authenticated_user_id or "").strip()
+        if not tenant_id:
+            raise _http_error(status.HTTP_400_BAD_REQUEST, "authenticated_user_required", "authenticated_user_id is required")
+        _assert_authenticated_tenant(tenant_id)
+
+        if hasattr(file, "read"):
+            raw = await file.read()
+        elif isinstance(file, (bytes, bytearray)):
+            raw = bytes(file)
+        else:
+            raise _http_error(status.HTTP_400_BAD_REQUEST, "invalid_backup_file", "Backup payload must be bytes or readable file")
+        try:
+            payload = json.loads(raw.decode("utf-8"))
+        except Exception as exc:
+            raise _http_error(status.HTTP_400_BAD_REQUEST, "invalid_backup_payload", "Backup payload must be valid UTF-8 JSON") from exc
+
+        restored_messages = 0
+        restored_ops = 0
+        try:
+            with connect_postgres() as conn:
+                execute_query(conn, "DELETE FROM messages WHERE tenant_id = %s", (tenant_id,))
+                execute_query(conn, "DELETE FROM oplog WHERE tenant_id = %s", (tenant_id,))
+                for row in payload.get("messages") or []:
+                    if not isinstance(row, dict):
+                        continue
+                    if str(row.get("tenant_id") or "").strip() != tenant_id:
+                        continue
+                    execute_query(
+                        conn,
+                        """
+                        INSERT INTO messages (tenant_id, dataset_id, message_id, sender_type, content, ts, user_id)
+                        VALUES (%s, %s, %s, %s, %s, %s, %s)
+                        """,
+                        (
+                            tenant_id,
+                            row.get("dataset_id"),
+                            row.get("message_id"),
+                            row.get("sender_type"),
+                            row.get("content"),
+                            row.get("ts") or _utc_now_iso(),
+                            row.get("user_id"),
+                        ),
+                    )
+                    restored_messages += 1
+                for row in payload.get("oplog") or []:
+                    if not isinstance(row, dict):
+                        continue
+                    if str(row.get("tenant_id") or "").strip() != tenant_id:
+                        continue
+                    execute_query(
+                        conn,
+                        """
+                        INSERT INTO oplog (tenant_id, dataset_id, op_id, op_type, payload_json, hlc_ts)
+                        VALUES (%s, %s, %s, %s, %s, %s)
+                        """,
+                        (
+                            tenant_id,
+                            row.get("dataset_id"),
+                            row.get("op_id") or str(uuid4()),
+                            row.get("op_type") or "restore_replay",
+                            row.get("payload_json") or "{}",
+                            row.get("hlc_ts") or _utc_now_iso(),
+                        ),
+                    )
+                    restored_ops += 1
+        except HTTPException:
+            raise
+        except PostgresConfigurationError as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_not_configured", str(exc)) from exc
+        except Exception as exc:
+            raise _http_error(status.HTTP_503_SERVICE_UNAVAILABLE, "postgres_restore_failed", str(exc)) from exc
+
+        return {
+            "status": "ok",
+            "tenant_id": tenant_id,
+            "restored_messages": restored_messages,
+            "restored_oplog_entries": restored_ops,
+        }
+
+
+class HostedDeviceService:
+    async def get_pairing_code(self) -> PairingCodeResponse:
+        raise NotImplementedError("HostedDeviceService not implemented yet")
+
+    async def pair_device(self, pairing_code: str, keep_existing_data: bool) -> PairDeviceResponse:
+        _ = (pairing_code, keep_existing_data)
+        raise NotImplementedError("HostedDeviceService not implemented yet")
+
+    async def get_device_info(self) -> DeviceInfoResponse:
+        raise NotImplementedError("HostedDeviceService not implemented yet")
+
+    async def set_device_name(self, device_name: str) -> DeviceNameResponse:
+        _ = device_name
+        raise NotImplementedError("HostedDeviceService not implemented yet")
+
+
+class HostedSyncService:
+    async def trigger_sync(self) -> SyncResponse:
+        raise NotImplementedError("HostedSyncService not implemented yet")

topos/sources/__init__.py

@@ -0,0 +1,6 @@
+"""Data source registry."""
+
+from .definitions import DataSourceDefinition
+from .registry import REGISTRY, list_sources
+
+__all__ = ["DataSourceDefinition", "REGISTRY", "list_sources"]

topos/sources/definitions.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from shared.filtering import filter_manifest_from_storage, get_filter_definition
+
+
+def _validate_field_transform_defaults(value: Optional[List[Dict[str, Any]]]) -> None:
+    """Ensure each transform_id in field_transform_defaults is in FILTER_CATALOG."""
+    if value is None or not value:
+        return
+    for i, entry in enumerate(value):
+        if not isinstance(entry, dict):
+            raise ValueError(f"field_transform_defaults[{i}] must be a dict")
+        table_id = entry.get("table_id")
+        field_name = entry.get("field")
+        transform_ids = entry.get("transform_ids")
+        if not isinstance(field_name, str) or not field_name.strip():
+            raise ValueError(f"field_transform_defaults[{i}] must have non-empty 'field'")
+        if not isinstance(transform_ids, list):
+            raise ValueError(f"field_transform_defaults[{i}] must have 'transform_ids' list")
+        for tid in transform_ids:
+            if not isinstance(tid, str) or not tid.strip():
+                raise ValueError(f"field_transform_defaults[{i}].transform_ids must contain non-empty strings")
+            if get_filter_definition(tid.strip()) is None:
+                raise ValueError(f"field_transform_defaults[{i}]: unknown transform_id {tid!r}")
+
+
+@dataclass(frozen=True)
+class DataSourceDefinition:
+    source_id: str
+    display_name: str
+    source_type: str
+    schema_id: str
+    parser_id: str
+    canonical_mapper_id: Optional[str] = None
+    canonical_group_id: Optional[str] = None
+    raw_enrichment_jobs: List[str] = field(default_factory=list)
+    canonical_enrichment_jobs: List[str] = field(default_factory=list)
+    analytics_profile_id: Optional[str] = None
+    enrichment_trigger: Literal["automatic", "manual"] = "automatic"
+    ingestion_trigger: Literal["automatic", "manual"] = "automatic"  # When to start processing after upload
+    # MVP roles/scopes (Sprint 02 Stage 1): scope this source's canonical output maps to
+    default_scope_id: str = "unknown"
+    allowed_scope_ids: Optional[List[str]] = None
+    default_filter_hints: Optional[List[str]] = None
+    filter_tier_kind: Optional[Literal["sensitivity", "inferability"]] = None
+    default_filter_tiers: Optional[Dict[str, Dict[str, object]]] = None
+    # Stage 10: registerer-defined field-level default transforms (which fields need which filters)
+    field_transform_defaults: Optional[List[Dict[str, Any]]] = None
+    # Managed source install metadata used by runtime ingestion for file sources.
+    tables: Optional[List[Dict[str, Any]]] = None
+    file_ingest_shape: Optional[Dict[str, Any]] = None
+    parser_column_map: Optional[Dict[str, str]] = None
+    canonical_mapping_connected: Optional[bool] = None
+    pipeline_include_parser: Optional[bool] = None
+    pipeline_include_data_table: Optional[bool] = None
+    pipeline_data_table_after_parser: Optional[bool] = None
+    pipeline_data_table_match_parser_output: Optional[bool] = None
+
+    def __post_init__(self) -> None:
+        if self.filter_tier_kind is not None and self.filter_tier_kind not in {"sensitivity", "inferability"}:
+            raise ValueError("filter_tier_kind must be 'sensitivity' or 'inferability'")
+        if self.default_filter_tiers is not None:
+            for tier_name, manifest in self.default_filter_tiers.items():
+                if tier_name not in {"low", "medium", "high"}:
+                    raise ValueError(f"Unsupported default filter tier {tier_name!r}")
+                filter_manifest_from_storage(manifest)
+        _validate_field_transform_defaults(self.field_transform_defaults)
+
+    def to_dict(self) -> dict:
+        out = {
+            "source_id": self.source_id,
+            "display_name": self.display_name,
+            "source_type": self.source_type,
+            "schema_id": self.schema_id,
+            "parser_id": self.parser_id,
+            "canonical_mapper_id": self.canonical_mapper_id,
+            "canonical_group_id": self.canonical_group_id,
+            "raw_enrichment_jobs": list(self.raw_enrichment_jobs),
+            "canonical_enrichment_jobs": list(self.canonical_enrichment_jobs),
+            "analytics_profile_id": self.analytics_profile_id,
+            "enrichment_trigger": self.enrichment_trigger,
+            "ingestion_trigger": self.ingestion_trigger,
+            "default_scope_id": self.default_scope_id,
+        }
+        if self.allowed_scope_ids is not None:
+            out["allowed_scope_ids"] = list(self.allowed_scope_ids)
+        if self.default_filter_hints is not None:
+            out["default_filter_hints"] = list(self.default_filter_hints)
+        if self.filter_tier_kind is not None:
+            out["filter_tier_kind"] = self.filter_tier_kind
+        if self.default_filter_tiers is not None:
+            out["default_filter_tiers"] = dict(self.default_filter_tiers)
+        if self.field_transform_defaults is not None:
+            out["field_transform_defaults"] = list(self.field_transform_defaults)
+        if self.tables is not None:
+            out["tables"] = list(self.tables)
+        if self.file_ingest_shape is not None:
+            out["file_ingest_shape"] = dict(self.file_ingest_shape)
+        if self.parser_column_map is not None:
+            out["parser_column_map"] = dict(self.parser_column_map)
+        if self.canonical_mapping_connected is not None:
+            out["canonical_mapping_connected"] = bool(self.canonical_mapping_connected)
+        if self.pipeline_include_parser is not None:
+            out["pipeline_include_parser"] = bool(self.pipeline_include_parser)
+        if self.pipeline_include_data_table is not None:
+            out["pipeline_include_data_table"] = bool(self.pipeline_include_data_table)
+        if self.pipeline_data_table_after_parser is not None:
+            out["pipeline_data_table_after_parser"] = bool(self.pipeline_data_table_after_parser)
+        if self.pipeline_data_table_match_parser_output is not None:
+            out["pipeline_data_table_match_parser_output"] = bool(self.pipeline_data_table_match_parser_output)
+        return out