syscred 2.2.0__py3-none-any.whl

syscred/backend_app.py

@@ -0,0 +1,363 @@
+# -*- coding: utf-8 -*-
+"""
+SysCRED Backend API - Flask Server
+===================================
+REST API for the credibility verification system.
+
+Endpoints:
+- POST /api/verify - Verify URL or text credibility
+- POST /api/seo - Get SEO analysis only
+- GET /api/ontology/stats - Get ontology statistics
+- GET /api/health - Health check
+- GET /api/config - View current configuration
+
+(c) Dominique S. Loyer - PhD Thesis Prototype
+"""
+
+import sys
+import os
+import traceback
+from flask import Flask, request, jsonify, send_from_directory
+from flask_cors import CORS
+
+# Add syscred package to path
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+# Import SysCRED modules
+try:
+    from syscred.verification_system import CredibilityVerificationSystem
+    from syscred.seo_analyzer import SEOAnalyzer
+    from syscred.ontology_manager import OntologyManager
+    from syscred.ontology_manager import OntologyManager
+    from syscred.config import config, Config
+    from syscred.database import init_db, db, AnalysisResult
+    SYSCRED_AVAILABLE = True
+    print("[SysCRED Backend] Modules imported successfully")
+except ImportError as e:
+    SYSCRED_AVAILABLE = False
+    print(f"[SysCRED Backend] Warning: Could not import modules: {e}")
+    # Define dummy init_db to prevent crash
+    def init_db(app): pass
+
+    # Fallback config
+    class Config:
+        HOST = "0.0.0.0"
+        PORT = 5000
+        DEBUG = True
+        ONTOLOGY_BASE_PATH = None
+        ONTOLOGY_DATA_PATH = None
+        LOAD_ML_MODELS = True
+        GOOGLE_FACT_CHECK_API_KEY = None
+    config = Config()
+
+# --- Initialize Flask App ---
+app = Flask(__name__)
+CORS(app)  # Enable CORS for frontend
+
+# Initialize Database
+try:
+    init_db(app) # [NEW] Setup DB connection
+except Exception as e:
+    print(f"[SysCRED Backend] Warning: DB init failed: {e}")
+
+# --- Initialize SysCRED System ---
+credibility_system = None
+seo_analyzer = None
+
+def initialize_system():
+    """Initialize the credibility system (lazy loading)."""
+    global credibility_system, seo_analyzer
+    
+    if not SYSCRED_AVAILABLE:
+        print("[SysCRED Backend] Cannot initialize - modules not available")
+        return False
+    
+    try:
+        # Initialize SEO analyzer (lightweight)
+        seo_analyzer = SEOAnalyzer()
+        print("[SysCRED Backend] SEO Analyzer initialized")
+        
+        # Initialize full system (may take time to load ML models)
+        print("[SysCRED Backend] Initializing credibility system (loading ML models)...")
+        ontology_base = str(config.ONTOLOGY_BASE_PATH) if config.ONTOLOGY_BASE_PATH else None
+        ontology_data = str(config.ONTOLOGY_DATA_PATH) if config.ONTOLOGY_DATA_PATH else None
+        credibility_system = CredibilityVerificationSystem(
+            ontology_base_path=ontology_base if ontology_base and os.path.exists(ontology_base) else None,
+            ontology_data_path=ontology_data,
+            load_ml_models=config.LOAD_ML_MODELS,
+            google_api_key=config.GOOGLE_FACT_CHECK_API_KEY
+        )
+        print("[SysCRED Backend] System initialized successfully!")
+        return True
+        
+    except Exception as e:
+        print(f"[SysCRED Backend] Error initializing system: {e}")
+        traceback.print_exc()
+        return False
+
+# --- API Routes ---
+
+@app.route('/')
+def index():
+    """Serve the frontend."""
+    return send_from_directory('static', 'index.html')
+
+
+@app.route('/api/health', methods=['GET'])
+def health_check():
+    """Health check endpoint."""
+    return jsonify({
+        'status': 'healthy',
+        'syscred_available': SYSCRED_AVAILABLE,
+        'system_initialized': credibility_system is not None,
+        'seo_analyzer_ready': seo_analyzer is not None
+    })
+
+
+@app.route('/api/verify', methods=['POST'])
+def verify_endpoint():
+    """
+    Main verification endpoint.
+    
+    Request JSON:
+    {
+        "input_data": "URL or text to verify",
+        "include_seo": true/false (optional, default true),
+        "include_pagerank": true/false (optional, default true)
+    }
+    """
+    global credibility_system
+    
+    # Lazy initialization
+    if credibility_system is None:
+        if not initialize_system():
+            return jsonify({
+                'error': 'System initialization failed. Check server logs.'
+            }), 503
+    
+    # Validate request
+    if not request.is_json:
+        return jsonify({'error': 'Request must be JSON'}), 400
+    
+    data = request.get_json()
+    input_data = data.get('input_data', '').strip()
+    
+    if not input_data:
+        return jsonify({'error': "'input_data' is required"}), 400
+    
+    include_seo = data.get('include_seo', True)
+    include_pagerank = data.get('include_pagerank', True)
+    
+    print(f"[SysCRED Backend] Verifying: {input_data[:100]}...")
+    
+    try:
+        # Run main verification
+        result = credibility_system.verify_information(input_data)
+        
+        if 'error' in result:
+            return jsonify(result), 400
+        
+        # Add SEO analysis if requested and it's a URL
+        if include_seo and credibility_system.is_url(input_data):
+            try:
+                web_content = credibility_system.api_clients.fetch_web_content(input_data)
+                if web_content.success:
+                    seo_result = seo_analyzer.analyze_seo(
+                        url=input_data,
+                        title=web_content.title,
+                        meta_description=web_content.meta_description,
+                        text_content=web_content.text_content
+                    )
+                    result['seoAnalysis'] = {
+                        'titleLength': seo_result.title_length,
+                        'titleHasKeywords': seo_result.title_has_keywords,
+                        'metaDescriptionLength': seo_result.meta_description_length,
+                        'wordCount': seo_result.word_count,
+                        'readabilityScore': round(seo_result.readability_score, 2),
+                        'seoScore': round(seo_result.seo_score, 2),
+                        'topKeywords': list(seo_result.keyword_density.keys())
+                    }
+            except Exception as e:
+                print(f"[SysCRED Backend] SEO analysis error: {e}")
+                result['seoAnalysis'] = {'error': str(e)}
+        
+        # Add PageRank estimation if requested
+        if include_pagerank and credibility_system.is_url(input_data):
+            try:
+                external_data = credibility_system.api_clients.fetch_external_data(input_data)
+                pr_result = seo_analyzer.estimate_pagerank(
+                    url=input_data,
+                    domain_age_days=external_data.domain_age_days,
+                    source_reputation=external_data.source_reputation
+                )
+                result['pageRankEstimation'] = {
+                    'estimatedPR': round(pr_result.estimated_pr, 3),
+                    'confidence': round(pr_result.confidence, 2),
+                    'factors': pr_result.factors,
+                    'explanation': pr_result.explanation_text
+                }
+            except Exception as e:
+                print(f"[SysCRED Backend] PageRank estimation error: {e}")
+                result['pageRankEstimation'] = {'error': str(e)}
+        
+        print(f"[SysCRED Backend] Score: {result.get('scoreCredibilite', 'N/A')}")
+        
+        # [NEW] Persist to Database
+        try:
+            new_analysis = AnalysisResult(
+                url=input_data[:500],
+                credibility_score=result.get('scoreCredibilite', 0.5),
+                summary=result.get('resumeAnalyse', ''),
+                source_reputation=result.get('detailsScore', {}).get('factors', [{}])[0].get('value')
+            )
+            db.session.add(new_analysis)
+            db.session.commit()
+            print(f"[SysCRED-DB] Result saved. ID: {new_analysis.id}")
+        except Exception as e:
+            print(f"[SysCRED-DB] Save failed: {e}")
+
+        return jsonify(result), 200
+        
+    except Exception as e:
+        print(f"[SysCRED Backend] Error: {e}")
+        traceback.print_exc()
+        return jsonify({'error': f'Internal error: {str(e)}'}), 500
+
+
+@app.route('/api/seo', methods=['POST'])
+def seo_endpoint():
+    """
+    SEO-only analysis endpoint (faster, no ML models needed).
+    
+    Request JSON:
+    {
+        "url": "URL to analyze"
+    }
+    """
+    global seo_analyzer
+    
+    if seo_analyzer is None:
+        seo_analyzer = SEOAnalyzer()
+    
+    if not request.is_json:
+        return jsonify({'error': 'Request must be JSON'}), 400
+    
+    data = request.get_json()
+    url = data.get('url', '').strip()
+    
+    if not url or not url.startswith('http'):
+        return jsonify({'error': 'Valid URL is required'}), 400
+    
+    try:
+        # Fetch content
+        from syscred.api_clients import ExternalAPIClients
+        api_client = ExternalAPIClients()
+        
+        web_content = api_client.fetch_web_content(url)
+        if not web_content.success:
+            return jsonify({'error': f'Failed to fetch URL: {web_content.error}'}), 400
+        
+        # SEO analysis
+        seo_result = seo_analyzer.analyze_seo(
+            url=url,
+            title=web_content.title,
+            meta_description=web_content.meta_description,
+            text_content=web_content.text_content
+        )
+        
+        # IR metrics
+        ir_metrics = seo_analyzer.get_ir_metrics(web_content.text_content)
+        
+        # PageRank estimation
+        external_data = api_client.fetch_external_data(url)
+        pr_result = seo_analyzer.estimate_pagerank(
+            url=url,
+            domain_age_days=external_data.domain_age_days,
+            source_reputation=external_data.source_reputation
+        )
+        
+        return jsonify({
+            'url': url,
+            'title': web_content.title,
+            'seo': {
+                'titleLength': seo_result.title_length,
+                'metaDescriptionLength': seo_result.meta_description_length,
+                'wordCount': seo_result.word_count,
+                'readabilityScore': round(seo_result.readability_score, 2),
+                'seoScore': round(seo_result.seo_score, 2),
+                'keywordDensity': seo_result.keyword_density
+            },
+            'irMetrics': {
+                'documentLength': ir_metrics.document_length,
+                'topTerms': ir_metrics.top_terms[:5],
+                'avgTermFrequency': round(ir_metrics.avg_term_frequency, 4)
+            },
+            'pageRank': {
+                'estimated': round(pr_result.estimated_pr, 3),
+                'confidence': round(pr_result.confidence, 2),
+                'factors': pr_result.factors
+            },
+            'domain': {
+                'reputation': external_data.source_reputation,
+                'ageDays': external_data.domain_age_days
+            }
+        }), 200
+        
+    except Exception as e:
+        print(f"[SysCRED Backend] SEO endpoint error: {e}")
+        traceback.print_exc()
+        return jsonify({'error': str(e)}), 500
+
+
+
+@app.route('/api/ontology/graph', methods=['GET'])
+def ontology_graph():
+    """Get ontology graph data for D3.js."""
+    global credibility_system
+    
+    if credibility_system and credibility_system.ontology_manager:
+        graph_data = credibility_system.ontology_manager.get_graph_json()
+        return jsonify(graph_data), 200
+    else:
+        # Return empty graph rather than 400 to avoid breaking frontend
+        return jsonify({'nodes': [], 'links': []}), 200
+
+
+@app.route('/api/ontology/stats', methods=['GET'])
+def ontology_stats():
+    """Get ontology statistics."""
+    global credibility_system
+    
+    if credibility_system and credibility_system.ontology_manager:
+        stats = credibility_system.ontology_manager.get_statistics()
+        return jsonify(stats), 200
+    else:
+        return jsonify({
+            'error': 'Ontology not loaded',
+            'base_triples': 0,
+            'data_triples': 0
+        }), 200
+
+
+# --- Main ---
+if __name__ == '__main__':
+    print("=" * 60)
+    print("SysCRED Backend API Server")
+    print("(c) Dominique S. Loyer - PhD Thesis Prototype")
+    print("=" * 60)
+    print()
+    
+    # Initialize system at startup
+    print("[SysCRED Backend] Pre-initializing system...")
+    initialize_system()
+    
+    print()
+    print("[SysCRED Backend] Starting Flask server...")
+    print("[SysCRED Backend] Endpoints:")
+    print("  - POST /api/verify     - Full credibility verification")
+    print("  - POST /api/seo        - SEO analysis only (faster)")
+    print("  - GET  /api/ontology/stats - Ontology statistics")
+    print("  - GET  /api/health     - Health check")
+    print()
+    
+    app.run(host='0.0.0.0', port=5001, debug=True)

syscred/config.py

@@ -0,0 +1,275 @@
+# -*- coding: utf-8 -*-
+"""
+SysCRED Configuration
+=====================
+Configuration centralisée pour le système de vérification de crédibilité.
+
+Usage:
+    from syscred.config import Config
+    
+    # Accéder aux paramètres
+    config = Config()
+    port = config.PORT
+    
+    # Ou avec variables d'environnement
+    # export SYSCRED_GOOGLE_API_KEY=your_key
+    # export SYSCRED_PORT=8080
+
+(c) Dominique S. Loyer - PhD Thesis Prototype
+"""
+
+import os
+from pathlib import Path
+from typing import Dict, Optional
+from dotenv import load_dotenv
+
+# Charger les variables depuis .env
+# Charger les variables depuis .env (Project Root)
+# Path: .../systemFactChecking/02_Code/syscred/config.py
+# Root .env is at .../systemFactChecking/.env (3 levels up)
+current_path = Path(__file__).resolve()
+env_path = current_path.parent.parent.parent / '.env'
+
+if not env_path.exists():
+    print(f"[Config] WARNING: .env not found at {env_path}")
+    # Try alternate location (sometimes CWD matters)
+    env_path = Path.cwd().parent / '.env'
+    
+load_dotenv(dotenv_path=env_path)
+print(f"[Config] Loading .env from {env_path}")
+print(f"[Config] SYSCRED_GOOGLE_API_KEY loaded: {'Yes' if os.environ.get('SYSCRED_GOOGLE_API_KEY') else 'No'}")
+
+
+
+class Config:
+    """
+    Configuration centralisée pour SysCRED.
+    
+    Les valeurs peuvent être override par des variables d'environnement
+    préfixées par SYSCRED_.
+    """
+    
+    # === Chemins ===
+    BASE_DIR = Path(__file__).parent.parent
+    ONTOLOGY_BASE_PATH = BASE_DIR / "sysCRED_onto26avrtil.ttl"
+    ONTOLOGY_DATA_PATH = BASE_DIR / "ontology" / "sysCRED_data.ttl"
+    
+    # === Serveur Flask ===
+    HOST = os.getenv("SYSCRED_HOST", "0.0.0.0")
+    PORT = int(os.getenv("SYSCRED_PORT", "5000"))
+    DEBUG = os.getenv("SYSCRED_DEBUG", "true").lower() == "true"
+    
+    # === API Keys ===
+    GOOGLE_FACT_CHECK_API_KEY = os.getenv("SYSCRED_GOOGLE_API_KEY")
+    DATABASE_URL = os.getenv("DATABASE_URL") # [NEW] Read DB URL from env
+    
+    # === Modèles ML ===
+    LOAD_ML_MODELS = os.getenv("SYSCRED_LOAD_ML", "true").lower() == "true"
+    SENTIMENT_MODEL = "distilbert-base-uncased-finetuned-sst-2-english"
+    NER_MODEL = "dbmdz/bert-large-cased-finetuned-conll03-english"
+    
+    # === Timeouts ===
+    WEB_FETCH_TIMEOUT = int(os.getenv("SYSCRED_TIMEOUT", "10"))
+    
+    # === Pondération des scores ===
+    SCORE_WEIGHTS = {
+        'source_reputation': 0.25,
+        'domain_age': 0.10,
+        'sentiment_neutrality': 0.15,
+        'entity_presence': 0.15,
+        'coherence': 0.15,
+        'fact_check': 0.20
+    }
+    
+    # === Seuils de crédibilité ===
+    CREDIBILITY_THRESHOLDS = {
+        'HIGH': 0.7,
+        'MEDIUM': 0.4,
+        'LOW': 0.0
+    }
+    
+    # === Base de données de réputation ===
+    # Les sources peuvent être étendues ou chargées d'un fichier externe
+    SOURCE_REPUTATIONS: Dict[str, str] = {
+        # === HAUTE CRÉDIBILITÉ ===
+        # Médias internationaux
+        'lemonde.fr': 'High',
+        'nytimes.com': 'High',
+        'reuters.com': 'High',
+        'bbc.com': 'High',
+        'bbc.co.uk': 'High',
+        'theguardian.com': 'High',
+        'apnews.com': 'High',
+        'afp.com': 'High',
+        'france24.com': 'High',
+        
+        # Médias canadiens
+        'cbc.ca': 'High',
+        'radio-canada.ca': 'High',
+        'lapresse.ca': 'High',
+        'ledevoir.com': 'High',
+        'theglobeandmail.com': 'High',
+        
+        # Sources académiques
+        'nature.com': 'High',
+        'sciencedirect.com': 'High',
+        'scholar.google.com': 'High',
+        'pubmed.ncbi.nlm.nih.gov': 'High',
+        'jstor.org': 'High',
+        'springer.com': 'High',
+        'ieee.org': 'High',
+        'acm.org': 'High',
+        'arxiv.org': 'High',
+        
+        # Fact-checkers
+        'factcheck.org': 'High',
+        'snopes.com': 'High',
+        'politifact.com': 'High',
+        'fullfact.org': 'High',
+        'checknews.fr': 'High',
+        
+        # Institutions
+        'who.int': 'High',
+        'un.org': 'High',
+        'europa.eu': 'High',
+        'canada.ca': 'High',
+        'gouv.fr': 'High',
+        'gouv.qc.ca': 'High',
+        
+        # === CRÉDIBILITÉ MOYENNE ===
+        'wikipedia.org': 'Medium',
+        'medium.com': 'Medium',
+        'huffpost.com': 'Medium',
+        'buzzfeed.com': 'Medium',
+        'vice.com': 'Medium',
+        'slate.com': 'Medium',
+        'theconversation.com': 'Medium',
+        
+        # === BASSE CRÉDIBILITÉ ===
+        'infowars.com': 'Low',
+        'naturalnews.com': 'Low',
+        'breitbart.com': 'Low',
+        'dailystormer.su': 'Low',
+        'beforeitsnews.com': 'Low',
+        'worldtruth.tv': 'Low',
+        'yournewswire.com': 'Low',
+    }
+    
+    # === Patterns de mésinformation ===
+    MISINFORMATION_KEYWORDS = [
+        'conspiracy', 'hoax', 'fake news', 'miracle cure', 
+        "they don't want you to know", 'mainstream media lies',
+        'deep state', 'plandemic', 'wake up sheeple',
+        'big pharma cover-up', 'government conspiracy',
+        'censored truth', 'what they hide'
+    ]
+    
+    @classmethod
+    def load_external_reputations(cls, filepath: str) -> None:
+        """
+        Charger des réputations supplémentaires depuis un fichier JSON.
+        
+        Args:
+            filepath: Chemin vers le fichier JSON avec format:
+                      {"domain.com": "High", "autre.com": "Low"}
+        """
+        import json
+        try:
+            with open(filepath, 'r') as f:
+                external_reps = json.load(f)
+                cls.SOURCE_REPUTATIONS.update(external_reps)
+                print(f"[Config] Loaded {len(external_reps)} external reputations")
+        except Exception as e:
+            print(f"[Config] Could not load external reputations: {e}")
+    
+    @classmethod
+    def update_weights(cls, new_weights: Dict[str, float]) -> None:
+        """
+        Mettre à jour les pondérations des scores.
+        
+        Args:
+            new_weights: Dictionnaire avec les nouvelles pondérations
+        """
+        cls.SCORE_WEIGHTS.update(new_weights)
+        # Normaliser pour que la somme = 1
+        total = sum(cls.SCORE_WEIGHTS.values())
+        cls.SCORE_WEIGHTS = {k: v/total for k, v in cls.SCORE_WEIGHTS.items()}
+        print(f"[Config] Updated weights: {cls.SCORE_WEIGHTS}")
+    
+    @classmethod
+    def to_dict(cls) -> Dict:
+        """Exporter la configuration actuelle en dictionnaire."""
+        return {
+            'host': cls.HOST,
+            'port': cls.PORT,
+            'debug': cls.DEBUG,
+            'google_api_configured': cls.GOOGLE_FACT_CHECK_API_KEY is not None,
+            'ml_models_enabled': cls.LOAD_ML_MODELS,
+            'score_weights': cls.SCORE_WEIGHTS,
+            'known_sources_count': len(cls.SOURCE_REPUTATIONS),
+            'ontology_base': str(cls.ONTOLOGY_BASE_PATH),
+            'ontology_data': str(cls.ONTOLOGY_DATA_PATH),
+        }
+    
+    @classmethod
+    def print_config(cls) -> None:
+        """Afficher la configuration actuelle."""
+        print("=" * 50)
+        print("SysCRED Configuration")
+        print("=" * 50)
+        for key, value in cls.to_dict().items():
+            print(f"  {key}: {value}")
+        print("=" * 50)
+
+
+# === Configuration par environnement ===
+
+class DevelopmentConfig(Config):
+    """Configuration pour développement local."""
+    DEBUG = True
+    LOAD_ML_MODELS = True
+
+
+class ProductionConfig(Config):
+    """Configuration pour production."""
+    DEBUG = False
+    LOAD_ML_MODELS = True
+    HOST = "0.0.0.0"
+
+
+class TestingConfig(Config):
+    """Configuration pour tests."""
+    DEBUG = True
+    LOAD_ML_MODELS = False  # Plus rapide pour les tests
+    WEB_FETCH_TIMEOUT = 5
+
+
+# Sélection automatique de la configuration
+def get_config() -> Config:
+    """
+    Retourne la configuration appropriée selon l'environnement.
+    
+    Variable d'environnement: SYSCRED_ENV (development, production, testing)
+    """
+    env = os.getenv("SYSCRED_ENV", "development").lower()
+    
+    configs = {
+        'development': DevelopmentConfig,
+        'production': ProductionConfig,
+        'testing': TestingConfig,
+    }
+    
+    return configs.get(env, DevelopmentConfig)
+
+
+# Instance par défaut
+config = get_config()
+
+
+if __name__ == "__main__":
+    # Test de la configuration
+    config.print_config()
+    
+    print("\n=== Source Reputations Sample ===")
+    for domain, rep in list(config.SOURCE_REPUTATIONS.items())[:10]:
+        print(f"  {domain}: {rep}")

syscred/database.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+Database Manager for SysCRED
+===========================
+Handles connection to Supabase (PostgreSQL) and defines models.
+"""
+
+import os
+from flask_sqlalchemy import SQLAlchemy
+from datetime import datetime
+
+# Initialize SQLAlchemy
+db = SQLAlchemy()
+
+class AnalysisResult(db.Model):
+    """Stores the result of a credibility analysis."""
+    __tablename__ = 'analysis_results'
+
+    id = db.Column(db.Integer, primary_key=True)
+    url = db.Column(db.String(500), nullable=False)
+    credibility_score = db.Column(db.Float, nullable=False)
+    summary = db.Column(db.Text)
+    created_at = db.Column(db.DateTime, default=datetime.utcnow)
+    
+    # Metadata stored as JSON if supported, or simplified columns
+    source_reputation = db.Column(db.String(50))
+    fact_check_count = db.Column(db.Integer, default=0)
+
+    def to_dict(self):
+        return {
+            'id': self.id,
+            'url': self.url,
+            'score': self.credibility_score,
+            'summary': self.summary,
+            'created_at': self.created_at.isoformat(),
+            'source_reputation': self.source_reputation
+        }
+
+def init_db(app):
+    """Initialize the database with the Flask app."""
+    # Fallback to sqlite for local dev if no DATABASE_URL
+    db_url = os.environ.get('DATABASE_URL')
+    if db_url and db_url.startswith("postgres://"):
+        db_url = db_url.replace("postgres://", "postgresql://", 1)
+    
+    app.config['SQLALCHEMY_DATABASE_URI'] = db_url or 'sqlite:///syscred.db'
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+    
+    db.init_app(app)
+    
+    # Create tables if they don't exist (basic migration)
+    with app.app_context():
+        db.create_all()
+        print("[SysCRED-DB] Database tables initialized.")