aiecs 1.0.1__py3-none-any.whl → 1.7.17__py3-none-any.whl

aiecs/scripts/knowledge_graph/run_threshold_experiments.py

@@ -0,0 +1,212 @@
+"""
+Script to run threshold sweep experiments for knowledge fusion matching.
+
+Usage:
+    poetry run python -m aiecs.scripts.knowledge_graph.run_threshold_experiments
+"""
+
+import asyncio
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import List
+
+# Add project root to path
+project_root = Path(__file__).parent.parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from aiecs.application.knowledge_graph.fusion.ab_testing import (
+    ABTestingFramework,
+    ExperimentResult,
+)
+from aiecs.application.knowledge_graph.fusion.evaluation_dataset import (
+    create_default_evaluation_dataset,
+)
+from aiecs.application.knowledge_graph.fusion.matching_config import (
+    FusionMatchingConfig,
+)
+from aiecs.application.knowledge_graph.fusion.similarity_pipeline import (
+    SimilarityPipeline,
+)
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+async def run_threshold_sweeps(
+    framework: ABTestingFramework,
+    output_dir: Path,
+) -> None:
+    """
+    Run threshold sweep experiments for all matching stages.
+
+    Args:
+        framework: ABTestingFramework instance
+        output_dir: Directory to save results
+    """
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Define threshold ranges for each stage
+    threshold_sweeps = {
+        "alias_match_score": [0.90, 0.92, 0.94, 0.96, 0.98, 1.0],
+        "abbreviation_match_score": [0.85, 0.88, 0.90, 0.92, 0.95, 0.98],
+        "normalization_match_score": [0.80, 0.85, 0.88, 0.90, 0.92, 0.95],
+        "semantic_threshold": [0.70, 0.75, 0.80, 0.85, 0.90, 0.95],
+        "string_similarity_threshold": [0.70, 0.75, 0.80, 0.85, 0.90],
+    }
+
+    all_results: List[ExperimentResult] = []
+
+    # Run sweeps for each threshold
+    for threshold_name, threshold_range in threshold_sweeps.items():
+        logger.info(f"Running sweep for {threshold_name}...")
+        results = await framework.threshold_sweep(
+            threshold_name=threshold_name,
+            threshold_range=threshold_range,
+        )
+        all_results.extend(results)
+
+        # Save individual sweep results
+        sweep_file = output_dir / f"sweep_{threshold_name}.json"
+        sweep_data = [
+            {
+                "config_name": r.config_name,
+                "threshold_value": getattr(r.config, threshold_name),
+                "metrics": r.metrics.to_dict(),
+                "stage_breakdown": r.stage_breakdown,
+            }
+            for r in results
+        ]
+        with open(sweep_file, "w") as f:
+            json.dump(sweep_data, f, indent=2)
+        logger.info(f"Saved sweep results to {sweep_file}")
+
+    # Evaluate default configuration
+    logger.info("Evaluating default configuration...")
+    default_config = FusionMatchingConfig()
+    default_result = await framework.evaluate_config("default", default_config)
+    all_results.append(default_result)
+
+    # Compare all results
+    comparison = framework.compare_results(all_results)
+    comparison_file = output_dir / "comparison.json"
+    with open(comparison_file, "w") as f:
+        json.dump(comparison, f, indent=2)
+    logger.info(f"Saved comparison results to {comparison_file}")
+
+    # Validate default thresholds
+    is_valid, validation = framework.validate_thresholds(
+        default_result, min_recall=0.90, min_precision=0.75
+    )
+    validation_file = output_dir / "validation.json"
+    with open(validation_file, "w") as f:
+        json.dump(validation, f, indent=2)
+    logger.info(f"Validation result: {'PASS' if is_valid else 'FAIL'}")
+    logger.info(f"Saved validation results to {validation_file}")
+
+    # Print summary
+    print("\n" + "=" * 80)
+    print("EXPERIMENT SUMMARY")
+    print("=" * 80)
+    print(f"\nDefault Configuration Metrics:")
+    print(f"  Recall:    {default_result.metrics.recall:.3f}")
+    print(f"  Precision: {default_result.metrics.precision:.3f}")
+    print(f"  F1 Score:  {default_result.metrics.f1_score:.3f}")
+    print(f"  Accuracy:  {default_result.metrics.accuracy:.3f}")
+    print(f"\nValidation: {'PASS' if is_valid else 'FAIL'}")
+    if not is_valid:
+        print(f"  Recall requirement: {validation['recall']:.3f} >= {validation['min_recall']:.3f} ({'✓' if validation['recall_met'] else '✗'})")
+        print(f"  Precision requirement: {validation['precision']:.3f} >= {validation['min_precision']:.3f} ({'✓' if validation['precision_met'] else '✗'})")
+    print("\n" + "=" * 80)
+
+
+async def run_domain_specific_experiments(
+    framework: ABTestingFramework,
+    output_dir: Path,
+) -> None:
+    """
+    Run experiments for domain-specific datasets.
+
+    Args:
+        framework: ABTestingFramework instance
+        output_dir: Directory to save results
+    """
+    domains = ["academic", "corporate", "medical"]
+    domain_results: dict = {}
+
+    for domain in domains:
+        logger.info(f"Running experiments for {domain} domain...")
+        domain_dataset = framework._dataset.get_by_domain(domain)
+
+        # Create domain-specific framework
+        domain_framework = ABTestingFramework(
+            pipeline=framework._pipeline,
+            dataset=domain_dataset,
+        )
+
+        # Evaluate default config on domain dataset
+        default_config = FusionMatchingConfig()
+        result = await domain_framework.evaluate_config(
+            f"default_{domain}", default_config
+        )
+
+        domain_results[domain] = {
+            "dataset_size": len(domain_dataset),
+            "metrics": result.metrics.to_dict(),
+            "stage_breakdown": result.stage_breakdown,
+        }
+
+    # Save domain-specific results
+    domain_file = output_dir / "domain_results.json"
+    with open(domain_file, "w") as f:
+        json.dump(domain_results, f, indent=2)
+    logger.info(f"Saved domain-specific results to {domain_file}")
+
+
+async def main():
+    """Main entry point."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run threshold sweep experiments for knowledge fusion"
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="experiment_results",
+        help="Output directory for results",
+    )
+    parser.add_argument(
+        "--domain-only",
+        action="store_true",
+        help="Only run domain-specific experiments",
+    )
+    args = parser.parse_args()
+
+    output_dir = Path(args.output_dir)
+
+    # Create evaluation dataset
+    logger.info("Creating evaluation dataset...")
+    dataset = create_default_evaluation_dataset()
+    logger.info(f"Dataset contains {len(dataset)} pairs")
+
+    # Initialize pipeline (without actual matchers for now - they're optional)
+    pipeline = SimilarityPipeline()
+
+    # Create framework
+    framework = ABTestingFramework(pipeline=pipeline, dataset=dataset)
+
+    if args.domain_only:
+        await run_domain_specific_experiments(framework, output_dir)
+    else:
+        await run_threshold_sweeps(framework, output_dir)
+        await run_domain_specific_experiments(framework, output_dir)
+
+    logger.info("Experiments completed!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

aiecs/scripts/migrations/multi_tenancy/README.md

@@ -0,0 +1,142 @@
+# Multi-Tenancy Migration Scripts
+
+This directory contains SQL migration scripts for adding multi-tenancy support to existing Knowledge Graph deployments.
+
+## Script Overview
+
+| Script | Description | Downtime | Rollback |
+|--------|-------------|----------|----------|
+| `001_add_tenant_id_column.sql` | Add tenant_id columns | No | `rollback_001_remove_tenant_id.sql` |
+| `002_backfill_tenant_id.sql` | Backfill existing data | No | `rollback_002_reset_tenant_id.sql` |
+| `003_create_tenant_indexes.sql` | Create performance indexes | No | `rollback_003_drop_tenant_indexes.sql` |
+| `004_enable_rls_policies.sql` | Enable RLS (optional) | No | `rollback_004_disable_rls.sql` |
+
+## Execution Order
+
+Run scripts in numerical order:
+
+```bash
+# PostgreSQL
+psql -U postgres -d knowledge_graph -f 001_add_tenant_id_column.sql
+psql -U postgres -d knowledge_graph -f 002_backfill_tenant_id.sql
+psql -U postgres -d knowledge_graph -f 003_create_tenant_indexes.sql
+psql -U postgres -d knowledge_graph -f 004_enable_rls_policies.sql  # Optional
+```
+
+## Pre-Migration Checklist
+
+- [ ] Backup database: `pg_dump knowledge_graph > backup.sql`
+- [ ] Test on staging environment first
+- [ ] Review and adjust tenant_id in backfill script (`legacy_default` vs `default`)
+- [ ] Verify disk space for indexes
+- [ ] Schedule migration during low-traffic period (for large databases)
+
+## Safety Features
+
+- All scripts use `IF NOT EXISTS` / `IF EXISTS` for idempotence
+- Script 003 uses `CREATE INDEX CONCURRENTLY` (no table locks)
+- Scripts include verification queries
+- Each script has corresponding rollback script
+
+## Rollback
+
+To rollback, run rollback scripts in reverse order:
+
+```bash
+psql -U postgres -d knowledge_graph -f rollback_004_disable_rls.sql
+psql -U postgres -d knowledge_graph -f rollback_003_drop_tenant_indexes.sql
+psql -U postgres -d knowledge_graph -f rollback_002_reset_tenant_id.sql
+psql -U postgres -d knowledge_graph -f rollback_001_remove_tenant_id.sql
+```
+
+Or restore from backup:
+
+```bash
+pg_restore -U postgres -d knowledge_graph -c backup.sql
+```
+
+## Monitoring Progress
+
+### Check Index Creation Progress
+
+```sql
+SELECT 
+    now()::time(0),
+    query,
+    state,
+    wait_event_type,
+    wait_event
+FROM pg_stat_activity 
+WHERE query LIKE '%CREATE INDEX%';
+```
+
+### Check Table Sizes
+
+```sql
+SELECT 
+    tablename,
+    pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size
+FROM pg_tables
+WHERE tablename IN ('graph_entities', 'graph_relations');
+```
+
+### Verify Migration Status
+
+```sql
+-- Check if tenant_id columns exist
+SELECT column_name, data_type, column_default
+FROM information_schema.columns 
+WHERE table_name IN ('graph_entities', 'graph_relations')
+  AND column_name = 'tenant_id';
+
+-- Check tenant distribution
+SELECT tenant_id, COUNT(*) 
+FROM graph_entities 
+GROUP BY tenant_id;
+
+-- Check if indexes exist
+SELECT indexname FROM pg_indexes 
+WHERE indexname LIKE '%tenant%';
+
+-- Check if RLS is enabled
+SELECT tablename, rowsecurity 
+FROM pg_tables 
+WHERE tablename IN ('graph_entities', 'graph_relations');
+```
+
+## Troubleshooting
+
+### Script Fails with "column already exists"
+
+The scripts are idempotent - this message means the migration was already run. Verify with:
+
+```sql
+\d graph_entities
+```
+
+### Index creation is slow
+
+Index creation time depends on table size. For very large tables (>10M rows), consider:
+
+- Running during maintenance window
+- Increasing `maintenance_work_mem`:
+  ```sql
+  SET maintenance_work_mem = '2GB';
+  ```
+
+### RLS causes performance issues
+
+If RLS adds unacceptable overhead, disable it and rely on application-level filtering:
+
+```bash
+psql -U postgres -d knowledge_graph -f rollback_004_disable_rls.sql
+```
+
+Or switch to SEPARATE_SCHEMA mode for better performance.
+
+## Support
+
+For detailed documentation:
+- [Migration Guide](../../../../docs/user/knowledge_graph/deployment/MULTI_TENANCY_MIGRATION.md)
+- [Setup Guide](../../../../docs/user/knowledge_graph/deployment/MULTI_TENANCY_GUIDE.md)
+- [Troubleshooting](../../../../docs/user/knowledge_graph/deployment/MULTI_TENANCY_TROUBLESHOOTING.md)