maxc-cli 0.1.0__py3-none-any.whl

maxc_cli/backend/meta.py

@@ -0,0 +1,312 @@
+"""Meta-related mixin for OdpsBackend."""
+
+from itertools import islice
+from typing import Any
+
+from ..config import TableColumn, TableDefinition
+from ..helpers import (
+    _dt_to_iso,
+    build_freshness_info,
+    build_latest_partition_info,
+    partition_spec_text,
+    record_to_dict,
+    translate_odps_error,
+)
+
+
+class MetaMixin:
+    """Mixin providing metadata methods."""
+
+    def list_tables(self) -> 'list[TableDefinition]':
+        """List tables in the current project."""
+        tables: 'list[TableDefinition]' = []
+        try:
+            for table in self.client.list_tables(project=self.project):
+                tables.append(self._table_stub(table))
+        except Exception as exc:
+            raise translate_odps_error(exc) from exc
+        return sorted(tables, key=lambda item: item.name)
+
+    def describe_table(self, table_name: 'str') -> 'TableDefinition':
+        """Describe a table with partitions and sample rows."""
+        table = self._get_table(table_name)
+        partitions = self._list_partitions(table, limit=20)
+        sample_rows = self._table_head(table, limit=2)
+        definition = self._table_definition_from_table(table)
+        definition.partitions = partitions
+        definition.sample_rows = sample_rows
+        return definition
+
+    def search_tables(self, keyword: 'str') -> 'list[dict[str, Any]]':
+        """Search tables by keyword."""
+        tokens = [item.lower() for item in keyword.split() if item.strip()] or [keyword.lower()]
+        matches: 'list[dict[str, Any]]' = []
+        for table in self.list_tables():
+            score = 0
+            searchable = f"{table.name} {table.description}".lower()
+            matched_columns: 'list[str]' = []
+            for token in tokens:
+                if token in searchable:
+                    score += 5
+            if score == 0:
+                for column in table.columns:
+                    text = f"{column.name} {column.comment}".lower()
+                    if any(token in text for token in tokens):
+                        score += 2
+                        matched_columns.append(column.name)
+            if score:
+                matches.append(
+                    {
+                        "table_name": table.name,
+                        "description": table.description,
+                        "score": score,
+                        "matched_columns": matched_columns,
+                    }
+                )
+        return sorted(matches, key=lambda item: (-item["score"], item["table_name"]))
+
+    def search_columns(self, keyword: 'str') -> 'list[dict[str, Any]]':
+        """Search columns by keyword."""
+        tokens = [item.lower() for item in keyword.split() if item.strip()] or [keyword.lower()]
+        matches: 'list[dict[str, Any]]' = []
+        for table in self.list_tables():
+            for column in table.columns:
+                score = 0
+                text = f"{column.name} {column.comment}".lower()
+                searchable = f"{table.name} {text}".lower()
+                for token in tokens:
+                    if token in column.name.lower():
+                        score += 8
+                    if token in text:
+                        score += 4
+                    if token in searchable:
+                        score += 2
+                if score:
+                    matches.append(
+                        {
+                            "table_name": table.name,
+                            "column_name": column.name,
+                            "type": column.type,
+                            "comment": column.comment,
+                            "score": score,
+                        }
+                    )
+        return sorted(matches, key=lambda item: (-item["score"], item["table_name"], item["column_name"]))
+
+    def latest_partition_info(self, table_name: 'str') -> 'tuple[dict[str, Any], list[str]]':
+        """Get latest partition info for a table."""
+        table = self._get_table(table_name)
+        definition = self._table_definition_from_table(table)
+        return self._latest_partition_info_from_table(table, definition)
+
+    def freshness_info(self, table_name: 'str') -> 'tuple[dict[str, Any], list[str]]':
+        """Get data freshness info for a table."""
+        table = self._get_table(table_name)
+        definition = self._table_definition_from_table(table)
+        latest_payload, warnings = self._latest_partition_info_from_table(table, definition)
+        return build_freshness_info(definition, latest_payload, warnings=warnings)
+
+    def lineage_info(self, table_name: 'str') -> 'tuple[dict[str, Any], list[str]]':
+        """Get table lineage info (placeholder - API not yet integrated)."""
+        table = self._get_table(table_name)
+        definition = self._table_definition_from_table(table)
+        return (
+            {
+                "table_name": definition.name,
+                "supported": False,
+                "lineage_source": "unavailable",
+                "coverage": "unsupported",
+                "upstream_tables": [],
+                "downstream_tables": [],
+                "limitation": "The current version does not integrate with the MaxCompute lineage API.",
+            },
+            ["The current version does not integrate with the MaxCompute lineage API, so lineage returns an explicit unsupported placeholder result."],
+        )
+
+    def list_projects(self) -> 'list[dict[str, Any]]':
+        """List all projects owned by the current user.
+
+        Note: This only returns basic info (name) to avoid triggering project.reload()
+        which requires Read permission on each project. Use get_project_info() for details.
+        """
+        projects: 'list[dict[str, Any]]' = []
+        try:
+            # 获取当前用户的 display name 作为 owner 过滤条件
+            owner = self._get_owner_display_name()
+            for project in self.client.list_projects(owner=owner):
+                # 只返回 list_projects 直接提供的基本信息
+                # 不要访问 comment, owner, properties 等属性，会触发 reload 需要 Read 权限
+                projects.append({
+                    "name": project.name,
+                })
+        except Exception as exc:
+            raise translate_odps_error(exc, "list_projects") from exc
+        return sorted(projects, key=lambda item: item["name"])
+
+    def list_schemas(self, *, project: 'str | None' = None) -> 'list[dict[str, Any]]':
+        """List all schemas in a project."""
+        target_project = project or self.project
+        schemas: 'list[dict[str, Any]]' = []
+        try:
+            for schema in self.client.list_schemas(project=target_project):
+                schemas.append({
+                    "name": schema.name,
+                })
+        except Exception as exc:
+            raise translate_odps_error(exc, "list_schemas") from exc
+        return sorted(schemas, key=lambda item: item["name"])
+
+    def get_project_info(self, project_name: 'str | None' = None) -> 'dict[str, Any]':
+        """Get detailed information about a project."""
+        target = project_name or self.project
+        try:
+            project = self.client.get_project(target)
+            # get_project 返回的对象需要 reload 才能获取完整属性
+            # 访问属性会自动触发 lazy loading
+            props = getattr(project, "properties", {}) or {}
+            extended_props = getattr(project, "extended_properties", {}) or {}
+
+            return {
+                "name": project.name,
+                "project_type": getattr(project, "type", None),
+                "comment": getattr(project, "comment", None),
+                "owner": getattr(project, "owner", None),
+                "state": getattr(project, "state", None) or getattr(project, "status", None),
+                "creation_time": _dt_to_iso(getattr(project, "creation_time", None)),
+                "last_modified_time": _dt_to_iso(getattr(project, "last_modified_time", None)),
+                "region": getattr(project, "region_id", None),
+                "allow_3_tier": props.get("allow3tier") or extended_props.get("allow3tier"),
+                "is_external_catalog_bound": props.get("isExternalCatalogBound") or extended_props.get("isExternalCatalogBound"),
+            }
+        except Exception as exc:
+            raise translate_odps_error(exc, "get_project_info") from exc
+
+    # Private methods for metadata handling
+
+    def _get_table(self, table_name: 'str', *, project: 'str | None' = None):
+        """Get ODPS table by name."""
+        try:
+            return self.client.get_table(table_name, project=project or self.project)
+        except Exception as exc:
+            raise translate_odps_error(exc) from exc
+
+    def _table_stub(self, table) -> 'TableDefinition':
+        """Create a minimal TableDefinition from table object (name only, no schema access)."""
+        return TableDefinition(
+            name=table.name,
+            description="",
+            columns=[],
+            sample_rows=[],
+            partitions=[],
+            upstream_tables=[],
+            downstream_tables=[],
+            partition_columns=[],
+            owner=None,
+            created_at=None,
+            updated_at=None,
+            table_type="TABLE",
+            size_bytes=None,
+        )
+
+    def _table_definition_from_table(self, table) -> 'TableDefinition':
+        """Create a full TableDefinition from ODPS table object."""
+        try:
+            columns = [
+                TableColumn(
+                    name=column.name,
+                    type=str(column.type),
+                    comment=getattr(column, "comment", "") or "",
+                )
+                for column in getattr(table.table_schema, "columns", [])
+            ]
+            partition_columns = [
+                TableColumn(
+                    name=column.name,
+                    type=str(column.type),
+                    comment=getattr(column, "comment", "") or "",
+                )
+                for column in getattr(table.table_schema, "partitions", [])
+            ]
+            return TableDefinition(
+                name=table.name,
+                description=getattr(table, "comment", "") or "",
+                columns=columns,
+                sample_rows=[],
+                partitions=[],
+                upstream_tables=[],
+                downstream_tables=[],
+                partition_columns=partition_columns,
+                owner=getattr(table, "owner", None),
+                created_at=_dt_to_iso(getattr(table, "creation_time", None)),
+                updated_at=_dt_to_iso(getattr(table, "last_data_modified_time", None)),
+                table_type="VIRTUAL_VIEW" if getattr(table, "is_virtual_view", False) else "TABLE",
+                size_bytes=(
+                    int(getattr(table, "size", 0))
+                    if getattr(table, "size", None) is not None
+                    else None
+                ),
+                extra_metadata={"lifecycle": getattr(table, "lifecycle", None)},
+            )
+        except Exception as exc:
+            raise translate_odps_error(exc) from exc
+
+    def _table_head(self, table, *, limit: 'int') -> 'list[dict[str, Any]]':
+        """Get first N rows from a table."""
+        try:
+            reader = table.head(limit)
+            rows = list(islice(reader, limit))
+        except Exception:
+            return []
+        columns = [column.name for column in table.table_schema.columns]
+        return [record_to_dict(columns, record.values) for record in rows]
+
+    def _list_partitions(self, table, *, limit: 'int') -> 'list[str]':
+        """List partition specs for a table."""
+        try:
+            from odps.errors import InvalidParameter as OdpsInvalidParameter
+            partitions = list(islice(table.iterate_partitions(), limit))
+        except Exception:
+            return []
+        return [str(partition.partition_spec) for partition in partitions]
+
+    def _latest_partition_info_from_table(
+        self,
+        table,
+        definition: 'TableDefinition',
+    ) -> 'tuple[dict[str, Any], list[str]]':
+        """Get latest partition info from ODPS table object."""
+        latest_partition = self._max_partition_spec(table)
+        if latest_partition:
+            return build_latest_partition_info(
+                definition,
+                source="odps_get_max_partition",
+                latest_partition_override=latest_partition,
+                visible_partition_count=None,
+            )
+
+        partitions = self._list_partitions(table, limit=200)
+        payload, warnings = build_latest_partition_info(
+            definition,
+            source="odps_iterate_partitions",
+            partitions=partitions,
+            visible_partition_count=len(partitions),
+        )
+        if definition.partition_columns and len(partitions) == 200:
+            warnings.append("Only the first 200 visible partitions were inspected. For very large tables, verify the result in the MaxCompute console as well.")
+        return payload, warnings
+
+    def _max_partition_spec(self, table) -> 'str | None':
+        """Get max partition spec from table using get_max_partition if available."""
+        getter = getattr(table, "get_max_partition", None)
+        if callable(getter):
+            for kwargs in ({"skip_empty": True}, {}):
+                try:
+                    partition = getter(**kwargs)
+                except TypeError:
+                    continue
+                except Exception:
+                    partition = None
+                text = partition_spec_text(partition)
+                if text:
+                    return text
+        return None

maxc_cli/backend/odps.py

@@ -0,0 +1,130 @@
+"""Main OdpsBackend class combining all mixins."""
+
+from itertools import islice
+from typing import Any
+
+from ..auth_providers import resolve_auth_connection
+from ..config import MaxCConfig
+from ..exceptions import PermissionDeniedError
+from ..helpers import (
+    _dt_to_iso,
+    record_to_dict,
+    translate_odps_error,
+)
+from ..models import QueryResult
+from ..utils import detect_operation, extract_table_names
+from .auth import AuthMixin
+from .data import DataMixin
+from .job import JobMixin
+from .meta import MetaMixin
+
+
+class OdpsBackend(
+    JobMixin,  # JobMixin extends QueryMixin
+    MetaMixin,
+    DataMixin,
+    AuthMixin,
+):
+    """MaxCompute backend for production use."""
+
+    supports_remote_jobs = True
+    supports_cost_check = False
+
+    def __init__(self, config: 'MaxCConfig') -> 'None':
+        """Initialize OdpsBackend with configuration."""
+        self.config = config
+        resolved = resolve_auth_connection(config)
+        self.resolved_auth = resolved
+        self.settings = resolved.settings
+        self.setting_sources = resolved.setting_sources
+        # Priority: config.default_project (includes session_override) > resolved.project
+        self.project = config.default_project or resolved.project
+        # Update resolved settings with the actual project being used
+        self.settings = dict(resolved.settings)
+        self.settings["project"] = self.project
+        self.client = resolved.create_client()
+        # 延迟获取 owner display name，避免不必要的 API 调用
+        self._owner_display_name: 'str | None' = None
+
+    def _validate_select(self, sql: 'str') -> 'None':
+        """Validate that SQL is a SELECT statement and allowed by config."""
+        operation = detect_operation(sql)
+        if operation not in self.config.allowed_operations:
+            raise PermissionDeniedError(
+                f"Configured allowed operations are limited to {', '.join(self.config.allowed_operations)}; received {operation}.",
+                suggestion="Update `allowed_operations` if you intentionally want to permit this operation.",
+            )
+        if operation != "SELECT":
+            raise PermissionDeniedError(f"This CLI currently supports only SELECT statements; received {operation}.")
+
+    def _instance_to_query_result(
+        self,
+        instance,
+        *,
+        project: 'str',
+        max_rows: 'int',
+        sql: 'str',
+        elapsed_ms: 'int',
+        offset: 'int' = 0,
+    ) -> 'QueryResult':
+        """Convert ODPS instance to QueryResult."""
+        try:
+            with instance.open_reader() as reader:
+                schema = [
+                    {
+                        "name": column.name,
+                        "type": str(column.type),
+                        "comment": "",
+                    }
+                    for column in reader.schema.columns
+                ]
+                rows = [
+                    record_to_dict(
+                        [column["name"] for column in schema],
+                        record.values,
+                    )
+                    for record in islice(reader, offset, offset + max_rows)
+                ]
+                total_rows = int(getattr(reader, "count", len(rows)) or len(rows))
+        except Exception as exc:
+            raise translate_odps_error(exc) from exc
+
+        bytes_scanned, extra_metadata = self._task_cost(instance)
+        returned_rows = len(rows)
+        has_more = total_rows > (offset + returned_rows)
+        extra_metadata["current_offset"] = offset
+
+        return QueryResult(
+            rows=rows,
+            schema=schema,
+            total_rows=total_rows,
+            returned_rows=returned_rows,
+            has_more=has_more,
+            next_cursor=None,  # cursor 由 app 层生成
+            elapsed_ms=elapsed_ms,
+            bytes_scanned=bytes_scanned,
+            project=project,
+            sql_executed=sql.rstrip(";"),
+            tables_used=extract_table_names(sql),
+            job_id=instance.id,
+            submitted_at=_dt_to_iso(getattr(instance, "start_time", None)),
+            completed_at=_dt_to_iso(getattr(instance, "end_time", None)),
+            extra_metadata=extra_metadata,
+        )
+
+    def _task_cost(self, instance) -> 'tuple[int | None, dict[str, Any]]':
+        """Get task cost from ODPS instance."""
+        try:
+            task_cost = instance.get_task_cost()
+        except Exception:
+            return None, {}
+        if task_cost is None:
+            return None, {}
+        return (
+            int(getattr(task_cost, "input_size", 0) or 0),
+            {
+                "task_cost_cpu": getattr(task_cost, "cpu_cost", None),
+                "task_cost_memory": getattr(task_cost, "memory_cost", None),
+                "estimated_input_size_bytes": getattr(task_cost, "input_size", None),
+            },
+        )

maxc_cli/backend/query.py

@@ -0,0 +1,148 @@
+"""Query-related mixin for OdpsBackend."""
+
+from time import monotonic
+from typing import Any
+
+from ..helpers import (
+    build_query_outline,
+    translate_odps_error,
+)
+from ..models import QueryResult
+from ..utils import extract_table_names, now_utc_iso
+
+
+class QueryMixin:
+    """Mixin providing query execution methods."""
+
+    def execute_query(
+        self,
+        sql: 'str',
+        *,
+        project: 'str',
+        max_rows: 'int',
+        dry_run: 'bool',
+        offset: 'int' = 0,
+        timeout: 'int | None' = None,
+    ) -> 'QueryResult':
+        """Execute a SQL query and return results.
+        
+        Args:
+            sql: SQL query to execute
+            project: Project name
+            max_rows: Maximum rows to return
+            dry_run: If True, only estimate cost without executing
+            offset: Row offset for pagination
+            timeout: Timeout in seconds (default: 300s / 5 minutes)
+        """
+        self._validate_select(sql)
+
+        started_at = now_utc_iso()
+        started_monotonic = monotonic()
+
+        if dry_run:
+            try:
+                sql_cost = self.client.execute_sql_cost(sql, project=project)
+            except Exception as exc:
+                raise translate_odps_error(exc) from exc
+            elapsed_ms = int((monotonic() - started_monotonic) * 1000)
+            return QueryResult(
+                rows=[],
+                schema=[],
+                total_rows=0,
+                returned_rows=0,
+                has_more=False,
+                next_cursor=None,
+                elapsed_ms=elapsed_ms,
+                bytes_scanned=int(sql_cost.input_size or 0),
+                project=project,
+                sql_executed=sql,
+                tables_used=extract_table_names(sql),
+                warnings=["MaxCompute dry-run returned SQLCost metadata and did not execute the query."],
+                submitted_at=started_at,
+                completed_at=now_utc_iso(),
+                extra_metadata={
+                    "sql_complexity": sql_cost.complexity,
+                    "sql_udf_num": sql_cost.udf_num,
+                    "estimated_input_size_bytes": sql_cost.input_size,
+                },
+            )
+
+        try:
+            instance = self.client.execute_sql(sql, project=project)
+            # Default timeout: 300 seconds (5 minutes) to prevent indefinite blocking
+            instance.wait_for_success(timeout=timeout or 300)
+        except Exception as exc:
+            raise translate_odps_error(exc) from exc
+
+        elapsed_ms = int((monotonic() - started_monotonic) * 1000)
+        result = self._instance_to_query_result(
+            instance,
+            project=project,
+            max_rows=max_rows,
+            sql=sql,
+            elapsed_ms=elapsed_ms,
+            offset=offset,
+        )
+        result.submitted_at = started_at
+        result.completed_at = now_utc_iso()
+        return result
+
+    def estimate_query_cost(self, sql: 'str', *, project: 'str') -> 'dict[str, Any]':
+        """Estimate the cost of a query."""
+        self._validate_select(sql)
+        started_monotonic = monotonic()
+        try:
+            sql_cost = self.client.execute_sql_cost(sql, project=project)
+        except Exception as exc:
+            raise translate_odps_error(exc) from exc
+        return {
+            **build_query_outline(sql),
+            "project": project,
+            "cost_model": "maxcompute_native_sql_cost",
+            "estimated_input_size_bytes": int(sql_cost.input_size or 0),
+            "task_cost_cpu": None,
+            "task_cost_memory": None,
+            "sql_complexity": sql_cost.complexity,
+            "sql_udf_num": sql_cost.udf_num,
+            "total_row_estimate": None,
+            "elapsed_ms": int((monotonic() - started_monotonic) * 1000),
+        }
+
+    def explain_query(self, sql: 'str', *, project: 'str') -> 'dict[str, Any]':
+        """Explain a query execution plan."""
+        estimate = self.estimate_query_cost(sql, project=project)
+        warnings = list(estimate.pop("warnings", []))
+        estimate["warnings"] = warnings
+        estimate["analysis_mode"] = "explain"
+        estimate["read_path"] = True
+        return estimate
+
+    def submit_query(
+        self,
+        sql: 'str',
+        *,
+        project: 'str',
+        idempotency_key: 'str | None' = None,
+    ):
+        """Submit a query for async execution."""
+        from ..models import JobInfo
+
+        try:
+            instance = self.client.execute_sql(
+                sql,
+                project=project,
+                unique_identifier_id=idempotency_key,
+            )
+        except Exception as exc:
+            raise translate_odps_error(exc) from exc
+        return JobInfo(
+            job_id=instance.id,
+            status="pending",
+            project=project,
+            progress=0,
+            sql=sql,
+            submitted_at=now_utc_iso(),
+            updated_at=now_utc_iso(),
+            logview=self._safe_logview(instance),
+            warnings=["The MaxCompute instance has been submitted; use job.status or job.wait to track it."],
+        )