relationalai 0.12.0__py3-none-any.whl → 0.12.2__py3-none-any.whl

relationalai/clients/direct_access_client.py

@@ -49,6 +49,11 @@ class DirectAccessClient:
             "suspend_engine": Endpoint(method="POST", endpoint="/v1alpha1/engines/{engine_type}/{engine_name}/suspend"),
             "resume_engine": Endpoint(method="POST", endpoint="/v1alpha1/engines/{engine_type}/{engine_name}/resume_async"),
             "prepare_index": Endpoint(method="POST", endpoint="/v1alpha1/index/prepare"),
+            "get_job": Endpoint(method="GET", endpoint="/v1alpha1/jobs/{job_type}/{job_id}"),
+            "list_jobs": Endpoint(method="GET", endpoint="/v1alpha1/jobs"),
+            "get_job_events": Endpoint(method="GET", endpoint="/v1alpha1/jobs/{job_type}/{job_id}/events/{stream_name}"),
+            "create_job": Endpoint(method="POST", endpoint="/v1alpha1/jobs"),
+            "cancel_job": Endpoint(method="POST", endpoint="/v1alpha1/jobs/{job_type}/{job_id}/cancel"),
         }
         self.http_session = self._create_retry_session()
 

relationalai/clients/snowflake.py

@@ -41,7 +41,7 @@ from ..clients.types import AvailableModel, EngineState, Import, ImportSource, I
 from ..clients.config import Config, ConfigStore, ENDPOINT_FILE
 from ..clients.client import Client, ExportParams, ProviderBase, ResourcesBase
 from ..clients.direct_access_client import DirectAccessClient
-from ..clients.util import IdentityParser, escape_for_f_string, get_pyrel_version, get_with_retries, poll_with_specified_overhead, safe_json_loads, sanitize_module_name, scrub_exception, wrap_with_request_id, ms_to_timestamp
+from ..clients.util import IdentityParser, escape_for_f_string, get_pyrel_version, get_with_retries, poll_with_specified_overhead, safe_json_loads, sanitize_module_name, scrub_exception, wrap_with_request_id, ms_to_timestamp, normalize_datetime
 from ..environments import runtime_env, HexEnvironment, SnowbookEnvironment
 from .. import dsl, rel, metamodel as m
 from ..errors import DuoSecurityFailed, EngineProvisioningFailed, EngineNameValidationException, EngineNotFoundException, EnginePending, EngineSizeMismatchWarning, EngineResumeFailed, Errors, InvalidAliasError, InvalidEngineSizeError, InvalidSourceTypeWarning, RAIAbortedTransactionError, RAIException, HexSessionException, SnowflakeAppMissingException, SnowflakeChangeTrackingNotEnabledException, SnowflakeDatabaseException, SnowflakeImportMissingException, SnowflakeInvalidSource, SnowflakeMissingConfigValuesException, SnowflakeProxyAPIDeprecationWarning, SnowflakeProxySourceError, SnowflakeRaiAppNotStarted, ModelNotFoundException, UnknownSourceWarning, ResponseStatusException, RowsDroppedFromTargetTableWarning, QueryTimeoutExceededException
@@ -761,13 +761,13 @@ Otherwise, remove it from your '{profile}' configuration profile.
         with debugging.span("create_model", name=name):
             self._exec(f"call {APP_NAME}.api.create_database('{name}', false, {debugging.gen_current_propagation_headers()});")
 
-    def delete_graph(self, name:str, force=False):
+    def delete_graph(self, name:str, force=False, language:str="rel"):
         prop_hdrs = debugging.gen_current_propagation_headers()
         if self.config.get("use_graph_index", USE_GRAPH_INDEX):
             keep_database = not force and self.config.get("reuse_model", True)
-            with debugging.span("release_index", name=name, keep_database=keep_database):
+            with debugging.span("release_index", name=name, keep_database=keep_database, language=language):
                 #TODO add headers to release_index
-                response = self._exec(f"call {APP_NAME}.api.release_index('{name}', OBJECT_CONSTRUCT('keep_database', {keep_database}));")
+                response = self._exec(f"call {APP_NAME}.api.release_index('{name}', OBJECT_CONSTRUCT('keep_database', {keep_database}, 'language', '{language}'));")
                 if response:
                     result = next(iter(response))
                     obj = json.loads(result["RELEASE_INDEX"])
@@ -795,6 +795,7 @@ Otherwise, remove it from your '{profile}' configuration profile.
         model: str,
         engine_name: str,
         engine_size: str | None = None,
+        language: str = "rel",
         program_span_id: str | None = None,
         headers: Dict | None = None,
     ):
@@ -805,6 +806,7 @@ Otherwise, remove it from your '{profile}' configuration profile.
             model,
             engine_name,
             engine_size,
+            language,
             program_span_id,
             headers,
             self.generation
@@ -1867,7 +1869,7 @@ Otherwise, remove it from your '{profile}' configuration profile.
         except Exception as e:
             err_message = str(e).lower()
             if _is_engine_issue(err_message):
-                self.auto_create_engine(engine)
+                self.auto_create_engine(engine, headers=headers)
                 self._exec_async_v2(
                     database, engine, raw_code_b64, inputs, readonly, nowait_durable,
                     headers=headers, bypass_index=bypass_index, language='lqp',
@@ -1907,7 +1909,7 @@ Otherwise, remove it from your '{profile}' configuration profile.
         except Exception as e:
             err_message = str(e).lower()
             if _is_engine_issue(err_message):
-                self.auto_create_engine(engine)
+                self.auto_create_engine(engine, headers=headers)
                 return self._exec_async_v2(
                     database,
                     engine,
@@ -1970,9 +1972,9 @@ Otherwise, remove it from your '{profile}' configuration profile.
                     if use_graph_index:
                         # we do not provide a default value for query_timeout_mins so that we can control the default on app level
                         if query_timeout_mins is not None:
-                            res = self._exec(f"call {APP_NAME}.api.exec_into_table(?, ?, ?, ?, ?, ?, ?, ?);", [database, engine, raw_code, output_table, readonly, nowait_durable, skip_invalid_data, query_timeout_mins])
+                            res = self._exec(f"call {APP_NAME}.api.exec_into_table(?, ?, ?, ?, ?, NULL, ?, {headers}, ?, ?);", [database, engine, raw_code, output_table, readonly, nowait_durable, skip_invalid_data, query_timeout_mins])
                         else:
-                            res = self._exec(f"call {APP_NAME}.api.exec_into_table(?, ?, ?, ?, ?, ?, ?);", [database, engine, raw_code, output_table, readonly, nowait_durable, skip_invalid_data])
+                             res = self._exec(f"call {APP_NAME}.api.exec_into_table(?, ?, ?, ?, ?, NULL, ?, {headers}, ?);", [database, engine, raw_code, output_table, readonly, nowait_durable, skip_invalid_data])
                         txn_id = json.loads(res[0]["EXEC_INTO_TABLE"])["rai_transaction_id"]
                         rejected_rows = json.loads(res[0]["EXEC_INTO_TABLE"]).get("rejected_rows", [])
                         rejected_rows_count = json.loads(res[0]["EXEC_INTO_TABLE"]).get("rejected_rows_count", 0)
@@ -2047,9 +2049,10 @@ Otherwise, remove it from your '{profile}' configuration profile.
         app_name = self.get_app_name()
 
         source_types = dict[str, SourceInfo]()
-        partitioned_sources: dict[str, dict[str, list[str]]] = defaultdict(
+        partitioned_sources: dict[str, dict[str, list[dict[str, str]]]] = defaultdict(
             lambda: defaultdict(list)
         )
+        fqn_to_parts: dict[str, tuple[str, str, str]] = {}
 
         for source in sources:
             parser = IdentityParser(source, True)
@@ -2057,82 +2060,219 @@ Otherwise, remove it from your '{profile}' configuration profile.
             assert len(parsed) == 4, f"Invalid source: {source}"
             db, schema, entity, identity = parsed
             assert db and schema and entity and identity, f"Invalid source: {source}"
-            source_types[identity] = cast(SourceInfo, {"type": None, "state": "", "columns_hash": None})
-            partitioned_sources[db][schema].append(entity)
-
-        # TODO: Move to NA layer
-        query = (
-            " UNION ALL ".join(
-                f"""SELECT
-                    inf.FQN,
-                    inf.KIND,
-                    inf.COLUMNS_HASH,
-                    IFF(DATEDIFF(second, ds.created_at::TIMESTAMP, inf.LAST_DDL::TIMESTAMP) > 0, 'STALE', 'CURRENT') AS STATE
-                FROM (
-                    SELECT (SELECT {app_name}.api.normalize_fq_ids(ARRAY_CONSTRUCT(FQ_OBJECT_NAME))[0]:identifier::string) as FQ_OBJECT_NAME,
-                        CREATED_AT FROM {app_name}.api.data_streams
-                    WHERE RAI_DATABASE = '{PYREL_ROOT_DB}'
-                ) ds
-                RIGHT JOIN (
+            source_types[identity] = cast(
+                SourceInfo,
+                {
+                    "type": None,
+                    "state": "",
+                    "columns_hash": None,
+                    "table_created_at": None,
+                    "stream_created_at": None,
+                    "last_ddl": None,
+                },
+            )
+            partitioned_sources[db][schema].append({"entity": entity, "identity": identity})
+            fqn_to_parts[identity] = (db, schema, entity)
+
+        if not partitioned_sources:
+            return source_types
+
+        state_queries: list[str] = []
+        for db, schemas in partitioned_sources.items():
+            select_rows: list[str] = []
+            for schema, tables in schemas.items():
+                for table_info in tables:
+                    select_rows.append(
+                        "SELECT "
+                        f"{IdentityParser.to_sql_value(db)} AS catalog_name, "
+                        f"{IdentityParser.to_sql_value(schema)} AS schema_name, "
+                        f"{IdentityParser.to_sql_value(table_info['entity'])} AS table_name"
+                    )
+
+            if not select_rows:
+                continue
+
+            target_entities_clause = "\n                        UNION ALL\n                        ".join(select_rows)
+            # Main query:
+            #   1. Enumerate the target tables via target_entities.
+            #   2. Pull their metadata (last_altered, type) from INFORMATION_SCHEMA.TABLES.
+            #   3. Look up the most recent stream activity for those FQNs only.
+            #   4. Capture creation timestamps and use last_ddl vs created_at to classify each target,
+            #      so we mark tables as stale when they were recreated even if column hashes still match.
+            state_queries.append(
+                f"""WITH target_entities AS (
+                        {target_entities_clause}
+                    ),
+                    table_info AS (
+                        SELECT
+                            {app_name}.api.normalize_fq_ids(
+                                ARRAY_CONSTRUCT(
+                                    CASE
+                                        WHEN t.table_catalog = UPPER(t.table_catalog) THEN t.table_catalog
+                                        ELSE '"' || t.table_catalog || '"'
+                                    END || '.' ||
+                                    CASE
+                                        WHEN t.table_schema = UPPER(t.table_schema) THEN t.table_schema
+                                        ELSE '"' || t.table_schema || '"'
+                                    END || '.' ||
+                                    CASE
+                                        WHEN t.table_name = UPPER(t.table_name) THEN t.table_name
+                                        ELSE '"' || t.table_name || '"'
+                                    END
+                                )
+                            )[0]:identifier::string AS fqn,
+                            CONVERT_TIMEZONE('UTC', t.last_altered) AS last_ddl,
+                            CONVERT_TIMEZONE('UTC', t.created) AS table_created_at,
+                            t.table_type AS kind
+                        FROM {db}.INFORMATION_SCHEMA.tables t
+                        JOIN target_entities te
+                            ON t.table_catalog = te.catalog_name
+                            AND t.table_schema = te.schema_name
+                            AND t.table_name = te.table_name
+                    ),
+                    stream_activity AS (
+                        SELECT
+                            sa.fqn,
+                            MAX(sa.created_at) AS created_at
+                        FROM (
+                            SELECT
+                                {app_name}.api.normalize_fq_ids(ARRAY_CONSTRUCT(fq_object_name))[0]:identifier::string AS fqn,
+                                created_at
+                            FROM {app_name}.api.data_streams
+                            WHERE rai_database = '{PYREL_ROOT_DB}'
+                        ) sa
+                        JOIN table_info ti
+                            ON sa.fqn = ti.fqn
+                        GROUP BY sa.fqn
+                    )
                     SELECT
-                        (SELECT {app_name}.api.normalize_fq_ids(
-                            ARRAY_CONSTRUCT(
-                                CASE
-                                    WHEN t.TABLE_CATALOG = UPPER(t.TABLE_CATALOG) THEN t.TABLE_CATALOG
-                                    ELSE '"' || t.TABLE_CATALOG || '"'
-                                END || '.' ||
-                                CASE
-                                    WHEN t.TABLE_SCHEMA = UPPER(t.TABLE_SCHEMA) THEN t.TABLE_SCHEMA
-                                    ELSE '"' || t.TABLE_SCHEMA || '"'
-                                END || '.' ||
-                                CASE
-                                    WHEN t.TABLE_NAME = UPPER(t.TABLE_NAME) THEN t.TABLE_NAME
-                                    ELSE '"' || t.TABLE_NAME || '"'
-                                END
-                            )
-                        )[0]:identifier::string) as FQN,
-                        CONVERT_TIMEZONE('UTC', LAST_DDL) AS LAST_DDL,
-                        TABLE_TYPE as KIND,
-                        SHA2(LISTAGG(
-                            COLUMN_NAME ||
-                            CASE
-                                WHEN c.NUMERIC_PRECISION IS NOT NULL AND c.NUMERIC_SCALE IS NOT NULL
-                                    THEN c.DATA_TYPE || '(' || c.NUMERIC_PRECISION || ',' || c.NUMERIC_SCALE || ')'
-                                WHEN c.DATETIME_PRECISION IS NOT NULL
-                                    THEN c.DATA_TYPE || '(0,' || c.DATETIME_PRECISION || ')'
-                                WHEN c.CHARACTER_MAXIMUM_LENGTH IS NOT NULL
-                                    THEN c.DATA_TYPE || '(' || c.CHARACTER_MAXIMUM_LENGTH || ')'
-                                ELSE c.DATA_TYPE
-                            END ||
-                            IS_NULLABLE,
-                            ','
-                        ) WITHIN GROUP (ORDER BY COLUMN_NAME), 256) as COLUMNS_HASH
-                    FROM {db}.INFORMATION_SCHEMA.TABLES t
-                    JOIN {db}.INFORMATION_SCHEMA.COLUMNS c
-                        ON t.TABLE_CATALOG = c.TABLE_CATALOG
-                        AND t.TABLE_SCHEMA = c.TABLE_SCHEMA
-                        AND t.TABLE_NAME = c.TABLE_NAME
-                    WHERE t.TABLE_CATALOG = {IdentityParser.to_sql_value(db)} AND ({" OR ".join(
-                        f"(t.TABLE_SCHEMA = {IdentityParser.to_sql_value(schema)} AND t.TABLE_NAME IN ({','.join(f'{IdentityParser.to_sql_value(table)}' for table in tables)}))"
-                        for schema, tables in schemas.items()
-                    )})
-                    GROUP BY t.TABLE_CATALOG, t.TABLE_SCHEMA, t.TABLE_NAME, t.LAST_DDL, t.TABLE_TYPE
-                ) inf on inf.FQN = ds.FQ_OBJECT_NAME
-            """
-                for db, schemas in partitioned_sources.items()
+                        ti.fqn,
+                        ti.kind,
+                        ti.last_ddl,
+                        ti.table_created_at,
+                        sa.created_at AS stream_created_at,
+                        IFF(
+                            DATEDIFF(second, sa.created_at::timestamp, ti.last_ddl::timestamp) > 0,
+                            'STALE',
+                            'CURRENT'
+                        ) AS state
+                    FROM table_info ti
+                    LEFT JOIN stream_activity sa
+                        ON sa.fqn = ti.fqn
+                """
             )
-            + ";"
+
+        stale_fqns: list[str] = []
+        for state_query in state_queries:
+            for row in self._exec(state_query):
+                row_dict = row.as_dict() if hasattr(row, "as_dict") else dict(row)
+                row_fqn = row_dict["FQN"]
+                parser = IdentityParser(row_fqn, True)
+                fqn = parser.identity
+                assert fqn, f"Error parsing returned FQN: {row_fqn}"
+
+                source_types[fqn]["type"] = (
+                    "TABLE" if row_dict["KIND"] == "BASE TABLE" else row_dict["KIND"]
+                )
+                source_types[fqn]["state"] = row_dict["STATE"]
+                source_types[fqn]["last_ddl"] = normalize_datetime(row_dict.get("LAST_DDL"))
+                source_types[fqn]["table_created_at"] = normalize_datetime(row_dict.get("TABLE_CREATED_AT"))
+                source_types[fqn]["stream_created_at"] = normalize_datetime(row_dict.get("STREAM_CREATED_AT"))
+                if row_dict["STATE"] == "STALE":
+                    stale_fqns.append(fqn)
+
+        if not stale_fqns:
+            return source_types
+
+        # We batch stale tables by database/schema so each Snowflake query can hash
+        # multiple objects at once instead of issuing one statement per table.
+        stale_partitioned: dict[str, dict[str, list[dict[str, str]]]] = defaultdict(
+            lambda: defaultdict(list)
         )
+        for fqn in stale_fqns:
+            db, schema, table = fqn_to_parts[fqn]
+            stale_partitioned[db][schema].append({"table": table, "identity": fqn})
+
+        # Build one hash query per database, grouping schemas/tables inside so we submit
+        # at most a handful of set-based statements to Snowflake.
+        for db, schemas in stale_partitioned.items():
+            column_select_rows: list[str] = []
+            for schema, tables in schemas.items():
+                for table_info in tables:
+                    # Build the literal rows for this db/schema so we can join back
+                    # against INFORMATION_SCHEMA.COLUMNS in a single statement.
+                    column_select_rows.append(
+                        "SELECT "
+                        f"{IdentityParser.to_sql_value(db)} AS catalog_name, "
+                        f"{IdentityParser.to_sql_value(schema)} AS schema_name, "
+                        f"{IdentityParser.to_sql_value(table_info['table'])} AS table_name"
+                    )
 
-        for row in self._exec(query):
-            row_fqn = row["FQN"]
-            parser = IdentityParser(row_fqn, True)
-            fqn = parser.identity
-            assert fqn, f"Error parsing returned FQN: {row_fqn}"
+            if not column_select_rows:
+                continue
 
-            source_types[fqn]["type"] = "TABLE" if row["KIND"] == "BASE TABLE" else row["KIND"]
-            source_types[fqn]["columns_hash"] = row["COLUMNS_HASH"]
-            source_types[fqn]["state"] = row["STATE"]
+            target_entities_clause = "\n                        UNION ALL\n                        ".join(column_select_rows)
+            # Main query: compute deterministic column hashes for every stale table
+            # in this database/schema batch so we can compare schemas without a round trip per table.
+            column_query = f"""WITH target_entities AS (
+                        {target_entities_clause}
+                    ),
+                    column_info AS (
+                        SELECT
+                            {app_name}.api.normalize_fq_ids(
+                                ARRAY_CONSTRUCT(
+                                    CASE
+                                        WHEN c.table_catalog = UPPER(c.table_catalog) THEN c.table_catalog
+                                        ELSE '"' || c.table_catalog || '"'
+                                    END || '.' ||
+                                    CASE
+                                        WHEN c.table_schema = UPPER(c.table_schema) THEN c.table_schema
+                                        ELSE '"' || c.table_schema || '"'
+                                    END || '.' ||
+                                    CASE
+                                        WHEN c.table_name = UPPER(c.table_name) THEN c.table_name
+                                        ELSE '"' || c.table_name || '"'
+                                    END
+                                )
+                            )[0]:identifier::string AS fqn,
+                            c.column_name,
+                            CASE
+                                WHEN c.numeric_precision IS NOT NULL AND c.numeric_scale IS NOT NULL
+                                    THEN c.data_type || '(' || c.numeric_precision || ',' || c.numeric_scale || ')'
+                                WHEN c.datetime_precision IS NOT NULL
+                                    THEN c.data_type || '(0,' || c.datetime_precision || ')'
+                                WHEN c.character_maximum_length IS NOT NULL
+                                    THEN c.data_type || '(' || c.character_maximum_length || ')'
+                                ELSE c.data_type
+                            END AS type_signature,
+                            IFF(c.is_nullable = 'YES', 'YES', 'NO') AS nullable_flag
+                        FROM {db}.INFORMATION_SCHEMA.COLUMNS c
+                        JOIN target_entities te
+                            ON c.table_catalog = te.catalog_name
+                            AND c.table_schema = te.schema_name
+                            AND c.table_name = te.table_name
+                    )
+                    SELECT
+                        fqn,
+                        HEX_ENCODE(
+                            HASH_AGG(
+                                HASH(
+                                    column_name,
+                                    type_signature,
+                                    nullable_flag
+                                )
+                            )
+                        ) AS columns_hash
+                    FROM column_info
+                    GROUP BY fqn
+                """
+
+            for row in self._exec(column_query):
+                row_fqn = row["FQN"]
+                parser = IdentityParser(row_fqn, True)
+                fqn = parser.identity
+                assert fqn, f"Error parsing returned FQN: {row_fqn}"
+                source_types[fqn]["columns_hash"] = row["COLUMNS_HASH"]
 
         return source_types
 
@@ -2142,12 +2282,13 @@ Otherwise, remove it from your '{profile}' configuration profile.
         invalid_sources = {}
         source_references = []
         for source, info in source_info.items():
-            if info['type'] is None:
+            source_type = info.get("type")
+            if source_type is None:
                 missing_sources.append(source)
-            elif info['type'] not in ("TABLE", "VIEW"):
-                invalid_sources[source] = info['type']
+            elif source_type not in ("TABLE", "VIEW"):
+                invalid_sources[source] = source_type
             else:
-                source_references.append(f"{app_name}.api.object_reference('{info['type']}', '{source}')")
+                source_references.append(f"{app_name}.api.object_reference('{source_type}', '{source}')")
 
         if missing_sources:
             current_role = self.get_sf_session().get_current_role()
@@ -2831,7 +2972,16 @@ class SnowflakeClient(Client):
 
         query_attrs_dict = json.loads(headers.get("X-Query-Attributes", "{}")) if headers else {}
         with debugging.span("poll_use_index", sources=self.resources.sources, model=model, engine=engine_name, **query_attrs_dict):
-            self.poll_use_index(app_name, self.resources.sources, model, engine_name, engine_size, program_span_id, headers=headers)
+            self.poll_use_index(
+                app_name=app_name,
+                sources=self.resources.sources,
+                model=model,
+                engine_name=engine_name,
+                engine_size=engine_size,
+                language="rel",
+                program_span_id=program_span_id,
+                headers=headers
+            )
 
         self.last_database_version = len(self.resources.sources)
         self._manage_packages()
@@ -2850,12 +3000,20 @@ class SnowflakeClient(Client):
         model: str,
         engine_name: str,
         engine_size: str | None = None,
+        language: str = "rel",
         program_span_id: str | None = None,
         headers: Dict | None = None,
     ):
         assert isinstance(self.resources, Resources)
         return self.resources.poll_use_index(
-            app_name, sources, model, engine_name, engine_size, program_span_id, headers=headers
+            app_name=app_name,
+            sources=sources,
+            model=model,
+            engine_name=engine_name,
+            engine_size=engine_size,
+            language=language,
+            program_span_id=program_span_id,
+            headers=headers
         )
 
 
@@ -3045,6 +3203,7 @@ class DirectAccessResources(Resources):
         headers: Dict[str, str] | None = None,
         path_params: Dict[str, str] | None = None,
         query_params: Dict[str, str] | None = None,
+        skip_auto_create: bool = False,
     ) -> requests.Response:
         with debugging.span("direct_access_request"):
             def _send_request():
@@ -3066,7 +3225,8 @@ class DirectAccessResources(Resources):
                         )
 
                     # fix engine on engine error and retry
-                    if _is_engine_issue(message):
+                    # Skip auto-retry if skip_auto_create is True to avoid recursion
+                    if _is_engine_issue(message) and not skip_auto_create:
                         engine = payload.get("engine_name", "") if payload else ""
                         self.auto_create_engine(engine)
                         response = _send_request()
@@ -3161,6 +3321,7 @@ class DirectAccessResources(Resources):
         model: str,
         engine_name: str,
         engine_size: str = "",
+        language: str = "rel",
         rai_relations: List[str] | None = None,
         pyrel_program_id: str | None  = None,
         skip_pull_relations: bool = False,
@@ -3176,6 +3337,7 @@ class DirectAccessResources(Resources):
             payload = {
                 "model_name": model,
                 "caller_engine_name": engine_name,
+                "language": language,
                 "pyrel_program_id": pyrel_program_id,
                 "skip_pull_relations": skip_pull_relations,
                 "rai_relations": rai_relations or [],
@@ -3201,6 +3363,7 @@ class DirectAccessResources(Resources):
         model: str,
         engine_name: str,
         engine_size: str | None = None,
+        language: str = "rel",
         program_span_id: str | None = None,
         headers: Dict | None = None,
     ):
@@ -3211,6 +3374,7 @@ class DirectAccessResources(Resources):
             model=model,
             engine_name=engine_name,
             engine_size=engine_size,
+            language=language,
             program_span_id=program_span_id,
             headers=headers,
             generation=self.generation,
@@ -3351,14 +3515,14 @@ class DirectAccessResources(Resources):
         with debugging.span("create_model", dbname=name):
             return self._create_database(name,"")
 
-    def delete_graph(self, name:str, force=False):
+    def delete_graph(self, name:str, force=False, language: str = "rel"):
         prop_hdrs = debugging.gen_current_propagation_headers()
         if self.config.get("use_graph_index", USE_GRAPH_INDEX):
             keep_database = not force and self.config.get("reuse_model", True)
-            with debugging.span("release_index", name=name, keep_database=keep_database):
+            with debugging.span("release_index", name=name, keep_database=keep_database, language=language):
                 response = self.request(
                     "release_index",
-                    payload={"model_name": name, "keep_database": keep_database},
+                    payload={"model_name": name, "keep_database": keep_database, "language": language},
                     headers=prop_hdrs,
                 )
                 if (
@@ -3431,7 +3595,7 @@ class DirectAccessResources(Resources):
         return sorted(engines, key=lambda x: x["name"])
 
     def get_engine(self, name: str):
-        response = self.request("get_engine", path_params={"engine_name": name, "engine_type": "logic"})
+        response = self.request("get_engine", path_params={"engine_name": name, "engine_type": "logic"}, skip_auto_create=True)
         if response.status_code == 404: # engine not found return 404
             return None
         elif response.status_code != 200:
@@ -3478,6 +3642,7 @@ class DirectAccessResources(Resources):
             payload=payload,
             path_params={"engine_type": "logic"},
             headers=headers,
+            skip_auto_create=True,
         )
         if response.status_code != 200:
             raise ResponseStatusException(
@@ -3489,6 +3654,7 @@ class DirectAccessResources(Resources):
             "delete_engine",
             path_params={"engine_name": name, "engine_type": "logic"},
             headers=headers,
+            skip_auto_create=True,
         )
         if response.status_code != 200:
             raise ResponseStatusException(
@@ -3499,6 +3665,7 @@ class DirectAccessResources(Resources):
         response = self.request(
             "suspend_engine",
             path_params={"engine_name": name, "engine_type": "logic"},
+            skip_auto_create=True,
         )
         if response.status_code != 200:
             raise ResponseStatusException(
@@ -3510,6 +3677,7 @@ class DirectAccessResources(Resources):
             "resume_engine",
             path_params={"engine_name": name, "engine_type": "logic"},
             headers=headers,
+            skip_auto_create=True,
         )
         if response.status_code != 200:
             raise ResponseStatusException(

relationalai/clients/types.py

@@ -38,10 +38,13 @@ class EngineState(TypedDict):
     auto_suspend: int|None
     suspends_at: datetime|None
 
-class SourceInfo(TypedDict):
+class SourceInfo(TypedDict, total=False):
     type: str|None
     state: str
     columns_hash: str|None
+    table_created_at: datetime|None
+    stream_created_at: datetime|None
+    last_ddl: datetime|None
     source: str