pvw-cli 1.2.8__py3-none-any.whl

purviewcli/client/config.py

@@ -0,0 +1,231 @@
+"""
+Configuration Management for  Purview CLI
+Handles environment configuration, profiles, and settings
+"""
+
+import os
+import json
+import yaml
+from pathlib import Path
+from typing import Dict, Any, Optional
+from dataclasses import dataclass, asdict
+from pydantic import BaseModel
+import logging
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class PurviewProfile:
+    """Purview connection profile"""
+    name: str
+    account_name: str
+    tenant_id: Optional[str] = None
+    client_id: Optional[str] = None
+    azure_region: Optional[str] = None
+    batch_size: int = 100
+    max_retries: int = 3
+    timeout: int = 30
+    default_collection: Optional[str] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'PurviewProfile':
+        return cls(**data)
+
+class ConfigManager:
+    """Manages CLI configuration and profiles"""
+    
+    def __init__(self, config_dir: Optional[str] = None):
+        self.config_dir = Path(config_dir) if config_dir else self._get_default_config_dir()
+        self.config_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.config_file = self.config_dir / 'config.yaml'
+        self.profiles_file = self.config_dir / 'profiles.yaml'
+        
+        self._config = self._load_config()
+        self._profiles = self._load_profiles()
+    
+    def _get_default_config_dir(self) -> Path:
+        """Get default configuration directory"""
+        if os.name == 'nt':  # Windows
+            config_dir = Path.home() / 'AppData' / 'Local' / 'purviewcli'
+        else:  # Unix-like
+            config_dir = Path.home() / '.config' / 'purviewcli'
+        return config_dir
+    
+    def _load_config(self) -> Dict[str, Any]:
+        """Load main configuration"""
+        if self.config_file.exists():
+            try:
+                with open(self.config_file, 'r') as f:
+                    return yaml.safe_load(f) or {}
+            except Exception as e:
+                logger.warning(f"Failed to load config: {e}")
+        
+        return {
+            'default_profile': None,
+            'debug': False,
+            'output_format': 'json',
+            'auto_update_check': True
+        }
+    
+    def _load_profiles(self) -> Dict[str, PurviewProfile]:
+        """Load connection profiles"""
+        profiles = {}
+        
+        if self.profiles_file.exists():
+            try:
+                with open(self.profiles_file, 'r') as f:
+                    data = yaml.safe_load(f) or {}
+                    for name, profile_data in data.items():
+                        profiles[name] = PurviewProfile.from_dict(profile_data)
+            except Exception as e:
+                logger.warning(f"Failed to load profiles: {e}")
+        
+        return profiles
+    
+    def save_config(self):
+        """Save configuration to file"""
+        try:
+            with open(self.config_file, 'w') as f:
+                yaml.dump(self._config, f, default_flow_style=False)
+        except Exception as e:
+            logger.error(f"Failed to save config: {e}")
+    
+    def save_profiles(self):
+        """Save profiles to file"""
+        try:
+            profiles_data = {name: profile.to_dict() for name, profile in self._profiles.items()}
+            with open(self.profiles_file, 'w') as f:
+                yaml.dump(profiles_data, f, default_flow_style=False)
+        except Exception as e:
+            logger.error(f"Failed to save profiles: {e}")
+    
+    def add_profile(self, profile: PurviewProfile) -> bool:
+        """Add or update a profile"""
+        try:
+            self._profiles[profile.name] = profile
+            self.save_profiles()
+            
+            # Set as default if it's the first profile
+            if len(self._profiles) == 1:
+                self.set_default_profile(profile.name)
+            
+            return True
+        except Exception as e:
+            logger.error(f"Failed to add profile: {e}")
+            return False
+    
+    def remove_profile(self, name: str) -> bool:
+        """Remove a profile"""
+        if name in self._profiles:
+            del self._profiles[name]
+            self.save_profiles()
+            
+            # Clear default if this was the default profile
+            if self._config.get('default_profile') == name:
+                self._config['default_profile'] = None
+                self.save_config()
+            
+            return True
+        return False
+    
+    def get_profile(self, name: Optional[str] = None) -> Optional[PurviewProfile]:
+        """Get a profile by name or default profile"""
+        if name is None:
+            name = self._config.get('default_profile')
+        
+        if name and name in self._profiles:
+            return self._profiles[name]
+        
+        return None
+    
+    def list_profiles(self) -> Dict[str, PurviewProfile]:
+        """List all profiles"""
+        return self._profiles.copy()
+    
+    def set_default_profile(self, name: str) -> bool:
+        """Set default profile"""
+        if name in self._profiles:
+            self._config['default_profile'] = name
+            self.save_config()
+            return True
+        return False
+    
+    def get_config(self, key: str, default=None):
+        """Get configuration value"""
+        return self._config.get(key, default)
+    
+    def set_config(self, key: str, value: Any):
+        """Set configuration value"""
+        self._config[key] = value
+        self.save_config()
+    
+    def resolve_account_name(self, account_name: Optional[str] = None, profile_name: Optional[str] = None) -> Optional[str]:
+        """Resolve account name from various sources"""
+        # 1. Explicit parameter
+        if account_name:
+            return account_name
+        
+        # 2. Profile
+        profile = self.get_profile(profile_name)
+        if profile:
+            return profile.account_name
+        
+        # 3. Environment variable
+        env_account = os.environ.get('PURVIEW_ACCOUNT_NAME')
+        if env_account:
+            return env_account
+        
+        return None
+    
+    def create_profile_from_env(self, name: str = 'default') -> Optional[PurviewProfile]:
+        """Create profile from environment variables"""
+        account_name = os.environ.get('PURVIEW_ACCOUNT_NAME')
+        if not account_name:
+            return None
+        
+        profile = PurviewProfile(
+            name=name,
+            account_name=account_name,
+            tenant_id=os.environ.get('AZURE_TENANT_ID'),
+            client_id=os.environ.get('AZURE_CLIENT_ID'),
+            azure_region=os.environ.get('AZURE_REGION'),
+            batch_size=int(os.environ.get('PURVIEW_BATCH_SIZE', '100')),
+            max_retries=int(os.environ.get('PURVIEW_MAX_RETRIES', '3')),
+            timeout=int(os.environ.get('PURVIEW_TIMEOUT', '30'))
+        )
+        
+        return profile
+
+class EnvironmentHelper:
+    """Helper for environment variable management"""
+    
+    @staticmethod
+    def setup_environment(profile: PurviewProfile):
+        """Setup environment variables from profile"""
+        os.environ['PURVIEW_ACCOUNT_NAME'] = profile.account_name
+        
+        if profile.tenant_id:
+            os.environ['AZURE_TENANT_ID'] = profile.tenant_id
+        
+        if profile.client_id:
+            os.environ['AZURE_CLIENT_ID'] = profile.client_id
+        
+        if profile.azure_region:
+            os.environ['AZURE_REGION'] = profile.azure_region
+    
+    @staticmethod
+    def get_auth_info() -> Dict[str, str]:
+        """Get authentication information"""
+        return {
+            'tenant_id': os.environ.get('AZURE_TENANT_ID', 'Not set'),
+            'client_id': os.environ.get('AZURE_CLIENT_ID', 'Not set'),
+            'region': os.environ.get('AZURE_REGION', 'public'),
+            'purview_account': os.environ.get('PURVIEW_ACCOUNT_NAME', 'Not set')
+        }
+
+# Global config manager instance
+config_manager = ConfigManager()

purviewcli/client/data_quality.py

@@ -0,0 +1,433 @@
+"""
+Data Quality and Validation Module
+Provides data quality checks and validation for Purview operations
+"""
+
+import pandas as pd
+import re
+from typing import Dict, List, Any, Optional, Tuple
+from dataclasses import dataclass
+from enum import Enum
+import logging
+
+logger = logging.getLogger(__name__)
+
+class ValidationSeverity(Enum):
+    ERROR = "error"
+    WARNING = "warning"
+    INFO = "info"
+
+@dataclass
+class ValidationRule:
+    """Data validation rule definition"""
+    name: str
+    description: str
+    severity: ValidationSeverity
+    column: Optional[str] = None
+    pattern: Optional[str] = None
+    min_length: Optional[int] = None
+    max_length: Optional[int] = None
+    required: bool = False
+    allowed_values: Optional[List[str]] = None
+    custom_validator: Optional[callable] = None
+
+@dataclass
+class ValidationResult:
+    """Result of a validation check"""
+    rule_name: str
+    severity: ValidationSeverity
+    message: str
+    row_index: Optional[int] = None
+    column: Optional[str] = None
+    value: Any = None
+
+class DataQualityValidator:
+    """Validates data quality for Purview operations"""
+    
+    def __init__(self):
+        self.rules = []
+        self._setup_default_rules()
+    
+    def _setup_default_rules(self):
+        """Setup default validation rules"""
+        # GUID validation
+        guid_pattern = r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
+        self.add_rule(ValidationRule(
+            name="valid_guid",
+            description="GUID format validation",
+            severity=ValidationSeverity.ERROR,
+            pattern=guid_pattern
+        ))
+        
+        # Email validation
+        email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+        self.add_rule(ValidationRule(
+            name="valid_email",
+            description="Email format validation",
+            severity=ValidationSeverity.WARNING,
+            pattern=email_pattern
+        ))
+        
+        # Qualified name validation
+        self.add_rule(ValidationRule(
+            name="qualified_name_format",
+            description="Qualified name should contain @ symbol",
+            severity=ValidationSeverity.ERROR,
+            custom_validator=lambda x: '@' in str(x) if x else False
+        ))
+        
+        # Name length validation
+        self.add_rule(ValidationRule(
+            name="name_length",
+            description="Name should be between 1 and 100 characters",
+            severity=ValidationSeverity.ERROR,
+            min_length=1,
+            max_length=100
+        ))
+    
+    def add_rule(self, rule: ValidationRule):
+        """Add a validation rule"""
+        self.rules.append(rule)
+    
+    def remove_rule(self, rule_name: str):
+        """Remove a validation rule"""
+        self.rules = [rule for rule in self.rules if rule.name != rule_name]
+    
+    def validate_dataframe(self, df: pd.DataFrame, column_rules: Dict[str, List[str]] = None) -> List[ValidationResult]:
+        """Validate entire DataFrame"""
+        results = []
+        
+        # Global validations
+        results.extend(self._validate_structure(df))
+        
+        # Column-specific validations
+        if column_rules:
+            for column, rule_names in column_rules.items():
+                if column in df.columns:
+                    for rule_name in rule_names:
+                        rule = self._get_rule(rule_name)
+                        if rule:
+                            results.extend(self._validate_column(df, column, rule))
+        
+        return results
+    
+    def validate_entity_data(self, entity_data: Dict[str, Any]) -> List[ValidationResult]:
+        """Validate entity data structure"""
+        results = []
+        
+        # Check required fields
+        required_fields = ['typeName']
+        for field in required_fields:
+            if field not in entity_data:
+                results.append(ValidationResult(
+                    rule_name="required_field",
+                    severity=ValidationSeverity.ERROR,
+                    message=f"Required field '{field}' is missing",
+                    column=field
+                ))
+        
+        # Validate attributes
+        attributes = entity_data.get('attributes', {})
+        if attributes:
+            results.extend(self._validate_entity_attributes(attributes))
+        
+        return results
+    
+    def _validate_structure(self, df: pd.DataFrame) -> List[ValidationResult]:
+        """Validate DataFrame structure"""
+        results = []
+        
+        # Check for empty DataFrame
+        if df.empty:
+            results.append(ValidationResult(
+                rule_name="empty_dataframe",
+                severity=ValidationSeverity.ERROR,
+                message="DataFrame is empty"
+            ))
+        
+        # Check for duplicate rows
+        duplicates = df.duplicated()
+        if duplicates.any():
+            duplicate_indices = df[duplicates].index.tolist()
+            results.append(ValidationResult(
+                rule_name="duplicate_rows",
+                severity=ValidationSeverity.WARNING,
+                message=f"Found {len(duplicate_indices)} duplicate rows at indices: {duplicate_indices}"
+            ))
+        
+        return results
+    
+    def _validate_column(self, df: pd.DataFrame, column: str, rule: ValidationRule) -> List[ValidationResult]:
+        """Validate specific column against rule"""
+        results = []
+        
+        for index, value in df[column].items():
+            result = self._validate_value(value, rule, index, column)
+            if result:
+                results.append(result)
+        
+        return results
+    
+    def _validate_value(self, value: Any, rule: ValidationRule, row_index: int = None, column: str = None) -> Optional[ValidationResult]:
+        """Validate single value against rule"""
+        # Skip validation for null values unless required
+        if pd.isna(value):
+            if rule.required:
+                return ValidationResult(
+                    rule_name=rule.name,
+                    severity=rule.severity,
+                    message=f"Required value is missing",
+                    row_index=row_index,
+                    column=column,
+                    value=value
+                )
+            return None
+        
+        str_value = str(value)
+        
+        # Pattern validation
+        if rule.pattern and not re.match(rule.pattern, str_value, re.IGNORECASE):
+            return ValidationResult(
+                rule_name=rule.name,
+                severity=rule.severity,
+                message=f"Value '{value}' does not match pattern {rule.pattern}",
+                row_index=row_index,
+                column=column,
+                value=value
+            )
+        
+        # Length validation
+        if rule.min_length and len(str_value) < rule.min_length:
+            return ValidationResult(
+                rule_name=rule.name,
+                severity=rule.severity,
+                message=f"Value '{value}' is too short (minimum {rule.min_length} characters)",
+                row_index=row_index,
+                column=column,
+                value=value
+            )
+        
+        if rule.max_length and len(str_value) > rule.max_length:
+            return ValidationResult(
+                rule_name=rule.name,
+                severity=rule.severity,
+                message=f"Value '{value}' is too long (maximum {rule.max_length} characters)",
+                row_index=row_index,
+                column=column,
+                value=value
+            )
+        
+        # Allowed values validation
+        if rule.allowed_values and str_value not in rule.allowed_values:
+            return ValidationResult(
+                rule_name=rule.name,
+                severity=rule.severity,
+                message=f"Value '{value}' is not in allowed values: {rule.allowed_values}",
+                row_index=row_index,
+                column=column,
+                value=value
+            )
+        
+        # Custom validator
+        if rule.custom_validator:
+            try:
+                if not rule.custom_validator(value):
+                    return ValidationResult(
+                        rule_name=rule.name,
+                        severity=rule.severity,
+                        message=f"Value '{value}' failed custom validation",
+                        row_index=row_index,
+                        column=column,
+                        value=value
+                    )
+            except Exception as e:
+                return ValidationResult(
+                    rule_name=rule.name,
+                    severity=ValidationSeverity.ERROR,
+                    message=f"Custom validator error: {e}",
+                    row_index=row_index,
+                    column=column,
+                    value=value
+                )
+        
+        return None
+    
+    def _validate_entity_attributes(self, attributes: Dict[str, Any]) -> List[ValidationResult]:
+        """Validate entity attributes"""
+        results = []
+        
+        # Validate qualifiedName format
+        qualified_name = attributes.get('qualifiedName')
+        if qualified_name:
+            rule = self._get_rule('qualified_name_format')
+            if rule:
+                result = self._validate_value(qualified_name, rule, column='qualifiedName')
+                if result:
+                    results.append(result)
+        
+        # Validate name length
+        name = attributes.get('name')
+        if name:
+            rule = self._get_rule('name_length')
+            if rule:
+                result = self._validate_value(name, rule, column='name')
+                if result:
+                    results.append(result)
+        
+        return results
+    
+    def _get_rule(self, rule_name: str) -> Optional[ValidationRule]:
+        """Get validation rule by name"""
+        for rule in self.rules:
+            if rule.name == rule_name:
+                return rule
+        return None
+
+class DataQualityReport:
+    """Generate data quality reports"""
+    
+    @staticmethod
+    def generate_report(validation_results: List[ValidationResult]) -> Dict[str, Any]:
+        """Generate comprehensive data quality report"""
+        
+        # Categorize results by severity
+        errors = [r for r in validation_results if r.severity == ValidationSeverity.ERROR]
+        warnings = [r for r in validation_results if r.severity == ValidationSeverity.WARNING]
+        info = [r for r in validation_results if r.severity == ValidationSeverity.INFO]
+        
+        # Count issues by rule
+        rule_counts = {}
+        for result in validation_results:
+            rule_counts[result.rule_name] = rule_counts.get(result.rule_name, 0) + 1
+        
+        # Count issues by column
+        column_counts = {}
+        for result in validation_results:
+            if result.column:
+                column_counts[result.column] = column_counts.get(result.column, 0) + 1
+        
+        return {
+            'summary': {
+                'total_issues': len(validation_results),
+                'errors': len(errors),
+                'warnings': len(warnings),
+                'info': len(info),
+                'data_quality_score': DataQualityReport._calculate_quality_score(validation_results)
+            },
+            'issues_by_rule': rule_counts,
+            'issues_by_column': column_counts,
+            'error_details': [
+                {
+                    'rule': r.rule_name,
+                    'message': r.message,
+                    'row': r.row_index,
+                    'column': r.column,
+                    'value': r.value
+                } for r in errors
+            ],
+            'warning_details': [
+                {
+                    'rule': r.rule_name,
+                    'message': r.message,
+                    'row': r.row_index,
+                    'column': r.column,
+                    'value': r.value
+                } for r in warnings
+            ]
+        }
+    
+    @staticmethod
+    def _calculate_quality_score(validation_results: List[ValidationResult]) -> float:
+        """Calculate data quality score (0-100)"""
+        if not validation_results:
+            return 100.0
+        
+        # Weight errors more heavily than warnings
+        error_weight = 3
+        warning_weight = 1
+        
+        total_score = sum(
+            error_weight if r.severity == ValidationSeverity.ERROR else warning_weight
+            for r in validation_results
+        )
+        
+        # Assume base score and deduct for issues
+        base_score = 100.0
+        deduction_per_issue = 2.0
+        
+        final_score = max(0.0, base_score - (total_score * deduction_per_issue))
+        return round(final_score, 1)
+    
+    @staticmethod
+    def export_report_to_csv(report: Dict[str, Any], output_file: str):
+        """Export validation report to CSV"""
+        
+        # Create detailed issues DataFrame
+        issues_data = []
+        
+        for error in report.get('error_details', []):
+            issues_data.append({
+                'severity': 'ERROR',
+                'rule': error['rule'],
+                'message': error['message'],
+                'row': error['row'],
+                'column': error['column'],
+                'value': error['value']
+            })
+        
+        for warning in report.get('warning_details', []):
+            issues_data.append({
+                'severity': 'WARNING',
+                'rule': warning['rule'],
+                'message': warning['message'],
+                'row': warning['row'],
+                'column': warning['column'],
+                'value': warning['value']
+            })
+        
+        if issues_data:
+            df = pd.DataFrame(issues_data)
+            df.to_csv(output_file, index=False)
+        else:
+            # Create empty file with headers
+            pd.DataFrame(columns=['severity', 'rule', 'message', 'row', 'column', 'value']).to_csv(output_file, index=False)
+
+# Predefined validation rule sets for common scenarios
+# Entity validation rules mapping - maps entity types to validation rule names
+ENTITY_VALIDATION_RULES = {
+    'dataset': [
+        'name_length',
+        'qualified_name_format', 
+        'valid_email',
+        'valid_guid'
+    ],
+    'table': [
+        'name_length',
+        'qualified_name_format',
+        'valid_email'
+    ],
+    'glossary_term': [
+        'name_length',
+        'valid_guid'
+    ]
+}
+
+# Legacy field-based validation rules (for backward compatibility)
+LEGACY_VALIDATION_RULES = {
+    'name': ['name_length'],
+    'qualifiedName': ['qualified_name_format'],
+    'owner': ['valid_email'],
+    'guid': ['valid_guid']
+}
+
+GLOSSARY_TERM_VALIDATION_RULES = {
+    'name': ['name_length'],
+    'glossaryGuid': ['valid_guid']
+}
+
+TABLE_VALIDATION_RULES = {
+    'name': ['name_length'],
+    'qualifiedName': ['qualified_name_format'],
+    'db': ['name_length'],
+    'owner': ['valid_email']
+}