ssci-subagent-skills 1.0.0

package/skills/processing-network-data/scripts/transform_data_format.py

@@ -0,0 +1,334 @@
+#!/usr/bin/env python
+"""
+数据格式转换工具
+在不同网络数据格式之间进行转换
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime
+from typing import Dict, List, Any
+
+def edgelist_to_adjacency(edges: List[Dict]) -> Dict:
+    """将边列表转换为邻接矩阵"""
+    # 收集所有节点
+    nodes = set()
+    for edge in edges:
+        nodes.add(edge.get('source', ''))
+        nodes.add(edge.get('target', ''))
+    nodes = [n for n in nodes if n]  # 移除空值
+    
+    # 创建节点映射
+    node_to_index = {node: i for i, node in enumerate(nodes)}
+    n = len(nodes)
+    
+    # 初始化矩阵
+    matrix = [[0.0 for _ in range(n)] for _ in range(n)]
+    
+    # 填充矩阵
+    for edge in edges:
+        source = edge.get('source', '')
+        target = edge.get('target', '')
+        weight = edge.get('weight', 1.0)
+        
+        if source in node_to_index and target in node_to_index:
+            i = node_to_index[source]
+            j = node_to_index[target]
+            
+            # 如果有多个边，使用最大权重
+            matrix[i][j] = max(matrix[i][j], weight)
+            matrix[j][i] = max(matrix[j][i], weight)  # 无向图
+    
+    return {
+        'matrix': matrix,
+        'node_labels': nodes,
+        'node_to_index': node_to_index,
+        'size': n,
+        'is_directed': False
+    }
+
+def adjacency_to_edgelist(adjacency: Dict) -> List[Dict]:
+    """将邻接矩阵转换为边列表"""
+    matrix = adjacency.get('matrix', [])
+    node_labels = adjacency.get('node_labels', [])
+    n = len(matrix)
+    
+    edges = []
+    
+    for i in range(n):
+        for j in range(i, n):  # 只处理上三角，避免重复
+            weight = matrix[i][j]
+            if weight > 0:  # 只保留有连接的边
+                edge = {
+                    'source': node_labels[i],
+                    'target': node_labels[j],
+                    'weight': weight
+                }
+                edges.append(edge)
+    
+    return edges
+
+def edgelist_to_adjlist(edges: List[Dict]) -> Dict:
+    """将边列表转换为邻接表"""
+    adjlist = {}
+    
+    for edge in edges:
+        source = edge.get('source', '')
+        target = edge.get('target', '')
+        weight = edge.get('weight', 1.0)
+        
+        if source not in adjlist:
+            adjlist[source] = []
+        if target not in adjlist:
+            adjlist[target] = []
+        
+        # 添加到邻接表
+        adjlist[source].append({
+            'neighbor': target,
+            'weight': weight,
+            'type': edge.get('type', '')
+        })
+        
+        # 无向图，添加反向连接
+        if source != target:
+            adjlist[target].append({
+                'neighbor': source,
+                'weight': weight,
+                'type': edge.get('type', '')
+            })
+    
+    return adjlist
+
+def adjlist_to_edgelist(adjlist: Dict) -> List[Dict]:
+    """将邻接表转换为边列表"""
+    edges = []
+    seen_edges = set()  # 用于去重
+    
+    for source, neighbors in adjlist.items():
+        for neighbor_info in neighbors:
+            target = neighbor_info.get('neighbor', '')
+            weight = neighbor_info.get('weight', 1.0)
+            edge_type = neighbor_info.get('type', '')
+            
+            # 创建边的唯一标识
+            edge_id = tuple(sorted([source, target]))
+            if edge_id not in seen_edges:
+                seen_edges.add(edge_id)
+                
+                edge = {
+                    'source': source,
+                    'target': target,
+                    'weight': weight,
+                    'type': edge_type
+                }
+                edges.append(edge)
+    
+    return edges
+
+def save_adjacency_matrix(adjacency: Dict, output_file: str):
+    """保存邻接矩阵到文件"""
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(adjacency, f, ensure_ascii=False, indent=2)
+
+def save_edgelist(edges: List[Dict], output_file: str):
+    """保存边列表到文件"""
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(edges, f, ensure_ascii=False, indent=2)
+
+def save_adjlist(adjlist: Dict, output_file: str):
+    """保存邻接表到文件"""
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(adjlist, f, ensure_ascii=False, indent=2)
+
+def validate_input_format(data: Dict, format_type: str) -> bool:
+    """验证输入格式"""
+    if format_type == 'edgelist':
+        # 检查edges是否在顶层或在details中
+        return ('edges' in data and isinstance(data['edges'], list)) or \
+               ('details' in data and 'edges' in data['details'] and isinstance(data['details']['edges'], list))
+    elif format_type == 'adjacency':
+        return ('adjacency_matrix' in data or 'matrix' in data) or \
+               ('details' in data and ('adjacency_matrix' in data['details'] or 'matrix' in data['details']))
+    elif format_type == 'adjlist':
+        return ('adjacency_list' in data and isinstance(data['adjacency_list'], dict)) or \
+               ('details' in data and 'adjacency_list' in data['details'] and isinstance(data['details']['adjacency_list'], dict))
+    return False
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='数据格式转换工具',
+        epilog='示例：python transform_data_format.py --input network.json --output adj.json --from edgelist --to adjacency'
+    )
+    
+    parser.add_argument('--input', '-i', required=True, help='输入数据文件（JSON）')
+    parser.add_argument('--output', '-o', required=True, help='输出文件')
+    parser.add_argument('--from', '-f', dest='from_format',
+                       choices=['edgelist', 'adjacency', 'adjlist'],
+                       required=True,
+                       help='输入格式')
+    parser.add_argument('--to', '-t', dest='to_format',
+                       choices=['edgelist', 'adjacency', 'adjlist'],
+                       required=True,
+                       help='输出格式')
+    
+    args = parser.parse_args()
+    
+    # 读取数据
+    try:
+        with open(args.input, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+    except Exception as e:
+        print(f"错误：无法读取输入文件 - {e}", file=sys.stderr)
+        sys.exit(1)
+    
+    start_time = datetime.now()
+    
+    # 验证输入格式
+    if not validate_input_format(data, args.from_format):
+        print(f"错误：输入数据格式不符合{args.from_format}要求", file=sys.stderr)
+        sys.exit(1)
+    
+    # 提取数据
+    if args.from_format == 'edgelist':
+        edges = data.get('edges', []) or data.get('details', {}).get('edges', [])
+        if not edges:
+            print("错误：未找到边列表数据", file=sys.stderr)
+            sys.exit(1)
+    elif args.from_format == 'adjacency':
+        adjacency = data.get('adjacency_matrix') or data.get('matrix', {}) or \
+                   data.get('details', {}).get('adjacency_matrix', {}) or \
+                   data.get('details', {}).get('matrix', {})
+        if not adjacency:
+            print("错误：未找到邻接矩阵数据", file=sys.stderr)
+            sys.exit(1)
+    elif args.from_format == 'adjlist':
+        adjlist = data.get('adjacency_list', {}) or data.get('details', {}).get('adjacency_list', {})
+        if not adjlist:
+            print("错误：未找到邻接表数据", file=sys.stderr)
+            sys.exit(1)
+    
+    # 执行转换
+    result = None
+    result_type = None
+    
+    # 边列表 -> 其他格式
+    if args.from_format == 'edgelist':
+        if args.to_format == 'adjacency':
+            result = edgelist_to_adjacency(edges)
+            result_type = 'adjacency_matrix'
+        elif args.to_format == 'adjlist':
+            result = edgelist_to_adjlist(edges)
+            result_type = 'adjacency_list'
+        elif args.to_format == 'edgelist':
+            result = edges
+            result_type = 'edges'
+    
+    # 邻接矩阵 -> 其他格式
+    elif args.from_format == 'adjacency':
+        if args.to_format == 'edgelist':
+            result = adjacency_to_edgelist(adjacency)
+            result_type = 'edges'
+        elif args.to_format == 'adjlist':
+            edges = adjacency_to_edgelist(adjacency)
+            result = edgelist_to_adjlist(edges)
+            result_type = 'adjacency_list'
+        elif args.to_format == 'adjacency':
+            result = adjacency
+            result_type = 'adjacency_matrix'
+    
+    # 邻接表 -> 其他格式
+    elif args.from_format == 'adjlist':
+        if args.to_format == 'edgelist':
+            result = adjlist_to_edgelist(adjlist)
+            result_type = 'edges'
+        elif args.to_format == 'adjacency':
+            edges = adjlist_to_edgelist(adjlist)
+            result = edgelist_to_adjacency(edges)
+            result_type = 'adjacency_matrix'
+        elif args.to_format == 'adjlist':
+            result = adjlist
+            result_type = 'adjacency_list'
+    
+    end_time = datetime.now()
+    
+    # 准备输出
+    if result_type == 'adjacency_matrix':
+        output_data = {
+            'summary': {
+                'matrix_size': result.get('size', 0),
+                'from_format': args.from_format,
+                'to_format': args.to_format,
+                'processing_time': round((end_time - start_time).total_seconds(), 2)
+            },
+            'details': {
+                result_type: result
+            },
+            'metadata': {
+                'input_file': args.input,
+                'timestamp': datetime.now().isoformat(),
+                'version': '1.0.0',
+                'skill': 'processing-network-data'
+            }
+        }
+    elif result_type == 'edges':
+        output_data = {
+            'summary': {
+                'n_edges': len(result),
+                'from_format': args.from_format,
+                'to_format': args.to_format,
+                'processing_time': round((end_time - start_time).total_seconds(), 2)
+            },
+            'details': {
+                result_type: result
+            },
+            'metadata': {
+                'input_file': args.input,
+                'timestamp': datetime.now().isoformat(),
+                'version': '1.0.0',
+                'skill': 'processing-network-data'
+            }
+        }
+    elif result_type == 'adjacency_list':
+        output_data = {
+            'summary': {
+                'n_nodes': len(result),
+                'from_format': args.from_format,
+                'to_format': args.to_format,
+                'processing_time': round((end_time - start_time).total_seconds(), 2)
+            },
+            'details': {
+                result_type: result
+            },
+            'metadata': {
+                'input_file': args.input,
+                'timestamp': datetime.now().isoformat(),
+                'version': '1.0.0',
+                'skill': 'processing-network-data'
+            }
+        }
+    
+    # 保存结果
+    try:
+        with open(args.output, 'w', encoding='utf-8') as f:
+            json.dump(output_data, f, ensure_ascii=False, indent=2)
+        
+        print(f"✓ 数据格式转换完成")
+        print(f"  - 源格式：{args.from_format}")
+        print(f"  - 目标格式：{args.to_format}")
+        
+        if result_type == 'adjacency_matrix':
+            print(f"  - 矩阵大小：{result.get('size', 0)}×{result.get('size', 0)}")
+        elif result_type == 'edges':
+            print(f"  - 边数：{len(result)}")
+        elif result_type == 'adjacency_list':
+            print(f"  - 节点数：{len(result)}")
+        
+        print(f"  - 输出文件：{args.output}")
+        
+    except Exception as e:
+        print(f"错误：无法保存结果 - {e}", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

package/skills/processing-network-data/scripts/validate_network_data.py

@@ -0,0 +1,322 @@
+#!/usr/bin/env python
+"""
+网络数据验证工具
+验证网络数据的完整性和有效性
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime
+from typing import Dict, List, Any, Set
+from collections import defaultdict
+
+def validate_edgelist(edges: List[Dict]) -> Dict:
+    """验证边列表"""
+    issues = []
+    warnings = []
+    
+    # 检查必需字段
+    required_fields = ['source', 'target']
+    for i, edge in enumerate(edges):
+        for field in required_fields:
+            if field not in edge or not edge[field]:
+                issues.append(f"边{i+1}: 缺少{field}字段")
+    
+    # 检查权重有效性
+    for i, edge in enumerate(edges):
+        weight = edge.get('weight')
+        if weight is not None:
+            try:
+                weight_val = float(weight)
+                if weight_val < 0 or weight_val > 1:
+                    warnings.append(f"边{i+1}: 权重{weight_val}超出[0,1]范围")
+            except (ValueError, TypeError):
+                issues.append(f"边{i+1}: 权重必须是数值")
+    
+    # 检查自环
+    self_loops = []
+    for i, edge in enumerate(edges):
+        if edge.get('source') == edge.get('target'):
+            self_loops.append(i+1)
+    
+    if self_loops:
+        warnings.append(f"发现{len(self_loops)}个自环边: {self_loops}")
+    
+    # 检查重复边
+    edge_pairs = []
+    duplicates = []
+    for i, edge in enumerate(edges):
+        pair = tuple(sorted([edge.get('source', ''), edge.get('target', '')]))
+        if pair in edge_pairs:
+            duplicates.append(i+1)
+        else:
+            edge_pairs.append(pair)
+    
+    if duplicates:
+        warnings.append(f"发现{len(duplicates)}个重复边: {duplicates}")
+    
+    return {
+        'issues': issues,
+        'warnings': warnings,
+        'n_edges': len(edges),
+        'n_self_loops': len(self_loops),
+        'n_duplicates': len(duplicates)
+    }
+
+def validate_node_attributes(attributes: Dict) -> Dict:
+    """验证节点属性"""
+    issues = []
+    warnings = []
+    
+    # 检查节点是否为空
+    empty_nodes = [node for node in attributes if not node]
+    if empty_nodes:
+        issues.append(f"发现{len(empty_nodes)}个空节点名")
+    
+    # 检查数值字段
+    numeric_fields = ['age', 'publications', 'h_index', 'grant_amount']
+    for node, attrs in attributes.items():
+        if node:  # 跳过空节点
+            for field in numeric_fields:
+                value = attrs.get(field)
+                if value is not None:
+                    try:
+                        val = float(value)
+                        # 检查合理性
+                        if field == 'age' and (val < 18 or val > 100):
+                            warnings.append(f"节点{node}: 年龄{val}可能不合理")
+                        elif field == 'publications' and val < 0:
+                            issues.append(f"节点{node}: 论文数不能为负")
+                        elif field == 'h_index' and val < 0:
+                            issues.append(f"节点{node}: H指数不能为负")
+                    except (ValueError, TypeError):
+                        issues.append(f"节点{node}: {field}必须是数值")
+    
+    # 检查分类字段
+    categorical_fields = ['gender', 'department', 'position']
+    valid_values = {
+        'gender': ['男', '女', '其他', ''],
+        'department': ['', '计算机科学', '软件工程', '人工智能', '数据科学'],
+        'position': ['', '教授', '副教授', '讲师', '研究员', '博士生', '硕士生', '博士后']
+    }
+    
+    for node, attrs in attributes.items():
+        if node:
+            for field in categorical_fields:
+                value = attrs.get(field, '')
+                if value and value not in valid_values.get(field, []):
+                    warnings.append(f"节点{node}: {field}值'{value}'不在常见选项中")
+    
+    return {
+        'issues': issues,
+        'warnings': warnings,
+        'n_nodes': len(attributes),
+        'n_empty_nodes': len(empty_nodes)
+    }
+
+def validate_network_consistency(edges: List[Dict], attributes: Dict) -> Dict:
+    """验证网络一致性"""
+    issues = []
+    warnings = []
+    
+    # 获取边中的所有节点
+    edge_nodes = set()
+    for edge in edges:
+        edge_nodes.add(edge.get('source', ''))
+        edge_nodes.add(edge.get('target', ''))
+    
+    # 获取属性中的所有节点
+    attr_nodes = set(attributes.keys())
+    
+    # 检查边中有但属性中没有的节点
+    missing_attrs = edge_nodes - attr_nodes
+    if missing_attrs:
+        warnings.append(f"边中存在但属性中缺失的节点: {list(missing_attrs)}")
+    
+    # 检查属性中有但边中没有的节点（孤立节点）
+    isolated_nodes = attr_nodes - edge_nodes
+    if isolated_nodes:
+        warnings.append(f"属性中存在但边中缺失的节点（孤立节点）: {list(isolated_nodes)}")
+    
+    # 检查连通性（简化版）
+    if len(edge_nodes) > 1:
+        # 构建邻接表
+        adjacency = defaultdict(set)
+        for edge in edges:
+            source = edge.get('source', '')
+            target = edge.get('target', '')
+            if source and target:
+                adjacency[source].add(target)
+                adjacency[target].add(source)
+        
+        # 简单的连通性检查
+        visited = set()
+        to_visit = [next(iter(edge_nodes))] if edge_nodes else []
+        
+        while to_visit:
+            node = to_visit.pop()
+            if node not in visited:
+                visited.add(node)
+                to_visit.extend(adjacency[node] - visited)
+        
+        if len(visited) < len(edge_nodes):
+            warnings.append(f"网络不连通，{len(edge_nodes)-len(visited)}个节点无法到达")
+    
+    return {
+        'issues': issues,
+        'warnings': warnings,
+        'n_edge_nodes': len(edge_nodes),
+        'n_attr_nodes': len(attr_nodes),
+        'n_missing_attrs': len(missing_attrs),
+        'n_isolated': len(isolated_nodes)
+    }
+
+def calculate_validation_score(validation_results: Dict) -> float:
+    """计算验证得分"""
+    total_issues = 0
+    total_warnings = 0
+    
+    # 统计问题和警告
+    for category in ['edgelist', 'node_attributes', 'network_consistency']:
+        if category in validation_results:
+            total_issues += len(validation_results[category].get('issues', []))
+            total_warnings += len(validation_results[category].get('warnings', []))
+    
+    # 计算得分（100分制）
+    # 每个问题扣10分，每个警告扣2分
+    score = 100 - (total_issues * 10) - (total_warnings * 2)
+    return max(0, score)
+
+def generate_validation_recommendations(validation_results: Dict) -> List[str]:
+    """生成验证建议"""
+    recommendations = []
+    
+    # 基于问题类型生成建议
+    for category, result in validation_results.items():
+        issues = result.get('issues', [])
+        warnings = result.get('warnings', [])
+        
+        if category == 'edgelist':
+            if '缺少' in str(issues):
+                recommendations.append("补充缺失的边字段信息")
+            if '权重' in str(issues):
+                recommendations.append("检查并修正权重值，确保在[0,1]范围内")
+            if '重复' in str(warnings):
+                recommendations.append("考虑合并重复边或保留最终权重")
+        
+        elif category == 'node_attributes':
+            if '数值' in str(issues):
+                recommendations.append("检查并修正数值字段的格式")
+            if '年龄' in str(warnings):
+                recommendations.append("核实年龄数据的准确性")
+            if '孤立' in str(warnings):
+                recommendations.append("检查是否需要包含孤立节点")
+        
+        elif category == 'network_consistency':
+            if '缺失' in str(warnings):
+                recommendations.append("为边中存在的节点补充属性信息")
+            if '不连通' in str(warnings):
+                recommendations.append("考虑网络是否应该是连通的")
+    
+    if not recommendations:
+        recommendations.append("数据质量良好，无明显问题")
+    
+    return recommendations
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='网络数据验证工具',
+        epilog='示例：python validate_network_data.py --input network.json --output validation.json'
+    )
+    
+    parser.add_argument('--input', '-i', required=True, help='输入网络数据文件（JSON）')
+    parser.add_argument('--output', '-o', default='validation_report.json', help='输出文件')
+    
+    args = parser.parse_args()
+    
+    # 读取数据
+    try:
+        with open(args.input, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+    except Exception as e:
+        print(f"错误：无法读取输入文件 - {e}", file=sys.stderr)
+        sys.exit(1)
+    
+    start_time = datetime.now()
+    
+    # 提取网络数据
+    details = data.get('details', {})
+    edges = details.get('edges', [])
+    attributes = details.get('node_attributes', {})
+    
+    if not edges and not attributes:
+        print("错误：未找到网络数据", file=sys.stderr)
+        sys.exit(1)
+    
+    # 执行验证
+    validation_results = {}
+    
+    # 验证边列表
+    if edges:
+        validation_results['edgelist'] = validate_edgelist(edges)
+    
+    # 验证节点属性
+    if attributes:
+        validation_results['node_attributes'] = validate_node_attributes(attributes)
+    
+    # 验证网络一致性
+    if edges and attributes:
+        validation_results['network_consistency'] = validate_network_consistency(edges, attributes)
+    
+    # 计算验证得分
+    validation_score = calculate_validation_score(validation_results)
+    
+    # 生成建议
+    recommendations = generate_validation_recommendations(validation_results)
+    
+    end_time = datetime.now()
+    
+    # 统计检查项
+    n_checks = sum(1 for result in validation_results.values() if result)
+    
+    # 准备输出
+    output = {
+        'summary': {
+            'is_valid': validation_score >= 80,  # 80分以上认为有效
+            'validation_score': round(validation_score, 2),
+            'n_checks': n_checks,
+            'n_issues': sum(len(r.get('issues', [])) for r in validation_results.values()),
+            'n_warnings': sum(len(r.get('warnings', [])) for r in validation_results.values()),
+            'processing_time': round((end_time - start_time).total_seconds(), 2)
+        },
+        'details': {
+            'validation_results': validation_results,
+            'recommendations': recommendations
+        },
+        'metadata': {
+            'input_file': args.input,
+            'timestamp': datetime.now().isoformat(),
+            'version': '1.0.0',
+            'skill': 'processing-network-data'
+        }
+    }
+    
+    # 保存结果
+    try:
+        with open(args.output, 'w', encoding='utf-8') as f:
+            json.dump(output, f, ensure_ascii=False, indent=2)
+        
+        print(f"✓ 网络数据验证完成")
+        print(f"  - 有效性：{'是' if output['summary']['is_valid'] else '否'}")
+        print(f"  - 验证得分：{validation_score:.2f}")
+        print(f"  - 问题数：{output['summary']['n_issues']}")
+        print(f"  - 警告数：{output['summary']['n_warnings']}")
+        print(f"  - 输出文件：{args.output}")
+        
+    except Exception as e:
+        print(f"错误：无法保存结果 - {e}", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()