diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..19923f3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,99 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [3.0.0] - 2024-12-15 + +### Highlights + +**DeepAudit v3.0.0** introduces a revolutionary **Multi-Agent Intelligent Audit System**: + +- Multi-Agent Architecture with Orchestrator-driven decision making +- RAG (Retrieval-Augmented Generation) knowledge base enhancement +- Docker sandbox for automated vulnerability verification +- Professional security tool integration + +### Added + +#### Multi-Agent Architecture +- **Orchestrator Agent**: Centralized orchestration for autonomous audit strategy decisions +- **Recon Agent**: Information gathering, technology stack identification, and entry point discovery +- **Analysis Agent**: Deep vulnerability analysis with Semgrep, RAG semantic search, and LLM analysis +- **Verification Agent**: Sandbox testing, PoC generation, false positive filtering + +#### RAG Knowledge Base +- Code semantic understanding with Tree-sitter AST-based chunking +- CWE/CVE vulnerability knowledge base integration +- Milvus/ChromaDB vector database support +- Multi-language support: Python, JavaScript, TypeScript, Java, Go, PHP, Rust + +#### Security Sandbox +- Docker isolated container for PoC execution +- Resource limits: memory, CPU constraints +- Network isolation with configurable access +- seccomp security policies + +#### Security Tools Integration +- **Semgrep**: Multi-language static analysis +- **Bandit**: Python security scanning +- **Gitleaks**: Secret leak detection +- **TruffleHog**: Deep secret scanning +- **npm audit**: Node.js dependency vulnerabilities +- **Safety**: Python dependency audit +- **OSV-Scanner**: Multi-language dependency vulnerabilities + +#### New Features +- Kunlun-M (MIT License) security scanner integration +- File upload size limit increased to 500MB with large file optimization +- Improved task tabs with card-style layout +- Enhanced error handling and project scope filtering +- Streaming LLM token usage reporting with input estimation + +### Changed +- Refactored Agent architecture with dynamic Agent tree +- Expanded high-risk file patterns and dangerous pattern library +- Enhanced sandbox functionality with forced sandbox verification +- Improved report generation with normalized severity comparisons +- Better agent stream stability preventing unnecessary reconnections + +### Fixed +- Agent stream stability issues with correct event buffer draining +- Sandbox tool initialization logging improvements +- Task phase update to REPORTING on completion +- Various UI/UX improvements in AgentAudit component + +--- + +## [2.0.0] - 2024-11-15 + +### Added +- Multi-LLM platform support (OpenAI, Claude, Gemini, Qwen, DeepSeek, Zhipu, etc.) +- Ollama local model support for privacy-focused deployments +- Project management with GitHub/GitLab import +- ZIP file upload support +- Instant code analysis feature +- What-Why-How three-step fix recommendations +- PDF/JSON report export +- Audit rules management (OWASP Top 10 built-in) +- Prompt template management with visual editor +- Runtime LLM configuration in browser +- i18n support (Chinese/English) + +### Changed +- Migrated to FastAPI backend +- React 18 frontend with TypeScript +- PostgreSQL database with Alembic migrations +- Docker Compose deployment support + +--- + +## [1.0.0] - 2024-10-01 + +### Added +- Initial release +- Basic code security audit functionality +- LLM-powered vulnerability detection +- Simple web interface diff --git a/README.md b/README.md index c6c6ad5..5d3df89 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,7 @@ cd docker/sandbox && ./build.sh | [Agent 审计](docs/AGENT_AUDIT.md) | Multi-Agent 审计模块详解 | | [配置说明](docs/CONFIGURATION.md) | 后端配置、审计规则、提示词模板 | | [LLM 平台支持](docs/LLM_PROVIDERS.md) | 各家 LLM 的配置方法和 API Key 获取 | +| [安全工具设置](docs/SECURITY_TOOLS_SETUP.md) | 安全扫描工具本地安装指南 | | [常见问题](docs/FAQ.md) | 遇到问题先看这里 | | [更新日志](CHANGELOG.md) | 版本更新记录 | | [贡献指南](CONTRIBUTING.md) | 想参与开发？看这个 | diff --git a/backend/app/services/agent/agents/base.py b/backend/app/services/agent/agents/base.py index 9f7c7be..9b0ced6 100644 --- a/backend/app/services/agent/agents/base.py +++ b/backend/app/services/agent/agents/base.py @@ -731,8 +731,9 @@ class BaseAgent(ABC): async def emit_tool_result(self, tool_name: str, result: str, duration_ms: int): """发射工具结果事件""" - # 🔥 将结果转换为字典格式，因为 AgentEventData.tool_output 期望 Dict 类型 - tool_output_dict = {"result": result[:2000] if result else ""} # 截断长输出 + # 🔥 修复：确保 result 不为 None，避免显示 "None" 字符串 + safe_result = result if result and result != "None" else "" + tool_output_dict = {"result": safe_result[:2000] if safe_result else ""} # 截断长输出 await self.emit_event( "tool_result", f"[{self.name}] 工具 {tool_name} 完成 ({duration_ms}ms)", @@ -1033,7 +1034,9 @@ class BaseAgent(ABC): result = await tool.execute(**tool_input) duration_ms = int((time.time() - start) * 1000) - await self.emit_tool_result(tool_name, str(result.data)[:200], duration_ms) + # 🔥 修复：确保传递有意义的结果字符串，避免 "None" + result_preview = str(result.data)[:200] if result.data is not None else (result.error[:200] if result.error else "") + await self.emit_tool_result(tool_name, result_preview, duration_ms) if result.success: output = str(result.data) diff --git a/backend/app/services/agent/tools/base.py b/backend/app/services/agent/tools/base.py index e0b8a10..7bada4b 100644 --- a/backend/app/services/agent/tools/base.py +++ b/backend/app/services/agent/tools/base.py @@ -91,9 +91,11 @@ class AgentTool(ABC): except Exception as e: logger.error(f"Tool '{self.name}' error: {e}", exc_info=True) + error_msg = str(e) result = ToolResult( success=False, - error=str(e), + data=f"工具执行异常: {error_msg}", # 🔥 修复：设置 data 字段避免 None + error=error_msg, ) duration_ms = int((time.time() - start_time) * 1000) diff --git a/backend/app/services/agent/tools/external_tools.py b/backend/app/services/agent/tools/external_tools.py index 01418c1..5a4731a 100644 --- a/backend/app/services/agent/tools/external_tools.py +++ b/backend/app/services/agent/tools/external_tools.py @@ -113,7 +113,12 @@ Semgrep 是业界领先的静态分析工具，支持 30+ 种编程语言。 # 确保 Docker 可用 await self.sandbox_manager.initialize() if not self.sandbox_manager.is_available: - return ToolResult(success=False, error=f"Semgrep unavailable: {self.sandbox_manager.get_diagnosis()}") + error_msg = f"Semgrep unavailable: {self.sandbox_manager.get_diagnosis()}" + return ToolResult( + success=False, + data=error_msg, # 🔥 修复：设置 data 字段避免 None + error=error_msg + ) # 构建命令 (相对于 /workspace) # 注意: target_path 是相对于 project_root 的 @@ -144,25 +149,46 @@ Semgrep 是业界领先的静态分析工具，支持 30+ 种编程语言。 timeout=300, network_mode="bridge" # 🔥 Semgrep 需要网络来下载规则 ) - + + # 🔥 添加调试日志 + logger.info(f"[Semgrep] 执行结果: success={result['success']}, exit_code={result['exit_code']}, " + f"stdout_len={len(result.get('stdout', ''))}, stderr_len={len(result.get('stderr', ''))}") + if result.get('error'): + logger.warning(f"[Semgrep] 错误信息: {result['error']}") + if result.get('stderr'): + logger.warning(f"[Semgrep] stderr: {result['stderr'][:500]}") + if not result["success"] and result["exit_code"] != 1: # 1 means findings were found + error_msg = result['stderr'][:500] or result['error'] or "未知错误" + logger.error(f"[Semgrep] 执行失败: {error_msg}") return ToolResult( success=False, - error=f"Semgrep 执行失败: {result['stderr'][:500] or result['error']}", + data=f"Semgrep 执行失败: {error_msg}", # 🔥 修复：设置 data 字段避免 None + error=f"Semgrep 执行失败: {error_msg}", ) - + # 解析结果 + stdout = result.get('stdout', '') try: # 尝试从 stdout 查找 JSON - json_start = result['stdout'].find('{') + json_start = stdout.find('{') + logger.debug(f"[Semgrep] JSON 起始位置: {json_start}, stdout 前200字符: {stdout[:200]}") + if json_start >= 0: - results = json.loads(result['stdout'][json_start:]) + json_str = stdout[json_start:] + results = json.loads(json_str) + logger.info(f"[Semgrep] JSON 解析成功, results 数量: {len(results.get('results', []))}") else: + logger.warning(f"[Semgrep] 未找到 JSON 起始符 '{{', stdout: {stdout[:500]}") results = {} - except json.JSONDecodeError: + except json.JSONDecodeError as e: + error_msg = f"无法解析 Semgrep 输出 (位置 {e.pos}): {e.msg}" + logger.error(f"[Semgrep] JSON 解析失败: {error_msg}") + logger.error(f"[Semgrep] 原始输出前500字符: {stdout[:500]}") return ToolResult( success=False, - error=f"无法解析 Semgrep 输出: {result['stdout'][:200]}", + data=error_msg, # 🔥 修复：设置 data 字段避免 None + error=error_msg, ) findings = results.get("results", [])[:max_results] @@ -204,9 +230,11 @@ Semgrep 是业界领先的静态分析工具，支持 30+ 种编程语言。 ) except Exception as e: + error_msg = f"Semgrep 执行错误: {str(e)}" return ToolResult( - success=False, - error=f"Semgrep 执行错误: {str(e)}" + success=False, + data=error_msg, # 🔥 修复：设置 data 字段避免 None + error=error_msg ) @@ -276,7 +304,8 @@ Bandit 是 Python 专用的安全分析工具，由 OpenStack 安全团队开发 # 确保 Docker 可用 await self.sandbox_manager.initialize() if not self.sandbox_manager.is_available: - return ToolResult(success=False, error=f"Bandit unavailable: {self.sandbox_manager.get_diagnosis()}") + error_msg = f"Bandit unavailable: {self.sandbox_manager.get_diagnosis()}" + return ToolResult(success=False, data=error_msg, error=error_msg) safe_target_path = target_path if not target_path.startswith("/") else target_path.lstrip("/") @@ -308,7 +337,8 @@ Bandit 是 Python 专用的安全分析工具，由 OpenStack 安全团队开发 else: results = {} except json.JSONDecodeError: - return ToolResult(success=False, error=f"无法解析 Bandit 输出: {result['stdout'][:200]}") + error_msg = f"无法解析 Bandit 输出: {result['stdout'][:200]}" + return ToolResult(success=False, data=error_msg, error=error_msg) findings = results.get("results", [])[:max_results] @@ -340,7 +370,8 @@ Bandit 是 Python 专用的安全分析工具，由 OpenStack 安全团队开发 ) except Exception as e: - return ToolResult(success=False, error=f"Bandit 执行错误: {str(e)}") + error_msg = f"Bandit 执行错误: {str(e)}" + return ToolResult(success=False, data=error_msg, error=error_msg) # ============ Gitleaks 工具 ============ @@ -409,7 +440,8 @@ Gitleaks 是专业的密钥检测工具，支持 150+ 种密钥类型。 # 确保 Docker 可用 await self.sandbox_manager.initialize() if not self.sandbox_manager.is_available: - return ToolResult(success=False, error=f"Gitleaks unavailable: {self.sandbox_manager.get_diagnosis()}") + error_msg = f"Gitleaks unavailable: {self.sandbox_manager.get_diagnosis()}" + return ToolResult(success=False, data=error_msg, error=error_msg) safe_target_path = target_path if not target_path.startswith("/") else target_path.lstrip("/") @@ -438,7 +470,7 @@ Gitleaks 是专业的密钥检测工具，支持 150+ 种密钥类型。 if result['exit_code'] != 0: # 🔥 修复：错误信息可能在 error 或 stderr 中 error_msg = result.get('error') or result.get('stderr', '')[:300] or '未知错误' - return ToolResult(success=False, error=f"Gitleaks 执行失败: {error_msg}") + return ToolResult(success=False, data=f"Gitleaks 执行失败: {error_msg}", error=f"Gitleaks 执行失败: {error_msg}") stdout = result['stdout'] @@ -497,7 +529,8 @@ Gitleaks 是专业的密钥检测工具，支持 150+ 种密钥类型。 ) except Exception as e: - return ToolResult(success=False, error=f"Gitleaks 执行错误: {str(e)}") + error_msg = f"Gitleaks 执行错误: {str(e)}" + return ToolResult(success=False, data=error_msg, error=error_msg) # ============ npm audit 工具 ============ @@ -551,7 +584,8 @@ class NpmAuditTool(AgentTool): # 确保 Docker 可用 await self.sandbox_manager.initialize() if not self.sandbox_manager.is_available: - return ToolResult(success=False, error=f"npm audit unavailable: {self.sandbox_manager.get_diagnosis()}") + error_msg = f"npm audit unavailable: {self.sandbox_manager.get_diagnosis()}" + return ToolResult(success=False, data=error_msg, error=error_msg) # 这里的 target_path 是相对于 project_root 的 # 防止空路径 @@ -564,9 +598,11 @@ class NpmAuditTool(AgentTool): # 宿主机预检查 package_json = os.path.join(full_path, "package.json") if not os.path.exists(package_json): + error_msg = f"未找到 package.json: {target_path}" return ToolResult( success=False, - error=f"未找到 package.json: {target_path}", + data=error_msg, + error=error_msg, ) cmd = ["npm", "audit", "--json"] @@ -643,7 +679,8 @@ class NpmAuditTool(AgentTool): ) except Exception as e: - return ToolResult(success=False, error=f"npm audit 错误: {str(e)}") + error_msg = f"npm audit 错误: {str(e)}" + return ToolResult(success=False, data=error_msg, error=error_msg) # ============ Safety 工具 (Python 依赖) ============ @@ -694,11 +731,13 @@ class SafetyTool(AgentTool): # 确保 Docker 可用 await self.sandbox_manager.initialize() if not self.sandbox_manager.is_available: - return ToolResult(success=False, error=f"Safety unavailable: {self.sandbox_manager.get_diagnosis()}") + error_msg = f"Safety unavailable: {self.sandbox_manager.get_diagnosis()}" + return ToolResult(success=False, data=error_msg, error=error_msg) full_path = os.path.join(self.project_root, requirements_file) if not os.path.exists(full_path): - return ToolResult(success=False, error=f"未找到依赖文件: {requirements_file}") + error_msg = f"未找到依赖文件: {requirements_file}" + return ToolResult(success=False, data=error_msg, error=error_msg) # commands # requirements_file relative path inside container is just requirements_file (assuming it's relative to root) @@ -766,7 +805,8 @@ class SafetyTool(AgentTool): ) except Exception as e: - return ToolResult(success=False, error=f"Safety 执行错误: {str(e)}") + error_msg = f"Safety 执行错误: {str(e)}" + return ToolResult(success=False, data=error_msg, error=error_msg) # ============ TruffleHog 工具 ============ @@ -823,7 +863,8 @@ TruffleHog 可以扫描代码和 Git 历史，并验证密钥是否有效。 # 确保 Docker 可用 await self.sandbox_manager.initialize() if not self.sandbox_manager.is_available: - return ToolResult(success=False, error=f"TruffleHog unavailable: {self.sandbox_manager.get_diagnosis()}") + error_msg = f"TruffleHog unavailable: {self.sandbox_manager.get_diagnosis()}" + return ToolResult(success=False, data=error_msg, error=error_msg) safe_target_path = target_path if not target_path.startswith("/") else target_path.lstrip("/") @@ -880,7 +921,8 @@ TruffleHog 可以扫描代码和 Git 历史，并验证密钥是否有效。 ) except Exception as e: - return ToolResult(success=False, error=f"TruffleHog 执行错误: {str(e)}") + error_msg = f"TruffleHog 执行错误: {str(e)}" + return ToolResult(success=False, data=error_msg, error=error_msg) # ============ OSV-Scanner 工具 ============ @@ -941,7 +983,8 @@ Google 开源的漏洞扫描工具，使用 OSV (Open Source Vulnerabilities) # 确保 Docker 可用 await self.sandbox_manager.initialize() if not self.sandbox_manager.is_available: - return ToolResult(success=False, error=f"OSV-Scanner unavailable: {self.sandbox_manager.get_diagnosis()}") + error_msg = f"OSV-Scanner unavailable: {self.sandbox_manager.get_diagnosis()}" + return ToolResult(success=False, data=error_msg, error=error_msg) safe_target_path = target_path if not target_path.startswith("/") else target_path.lstrip("/") @@ -995,7 +1038,8 @@ Google 开源的漏洞扫描工具，使用 OSV (Open Source Vulnerabilities) ) except Exception as e: - return ToolResult(success=False, error=f"OSV-Scanner 执行错误: {str(e)}") + error_msg = f"OSV-Scanner 执行错误: {str(e)}" + return ToolResult(success=False, data=error_msg, error=error_msg) # ============ 导出所有工具 ============ diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 7066ce7..cc93d8a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -2,7 +2,7 @@ name = "deepaudit-backend" version = "3.0.0" description = "DeepAudit Backend API - AI-Powered Code Security Audit Platform" -requires-python = ">=3.11,<3.13" +requires-python = ">=3.11" readme = "README.md" license = { text = "MIT" } authors = [ @@ -108,9 +108,11 @@ docs = [ ] [project.urls] -Homepage = "https://github.com/your-org/deepaudit" -Documentation = "https://docs.deepaudit.io" -Repository = "https://github.com/your-org/deepaudit" +Homepage = "https://github.com/lintsinghua/DeepAudit" +Documentation = "https://github.com/lintsinghua/DeepAudit/tree/main/docs" +Repository = "https://github.com/lintsinghua/DeepAudit" +Issues = "https://github.com/lintsinghua/DeepAudit/issues" +Changelog = "https://github.com/lintsinghua/DeepAudit/blob/main/CHANGELOG.md" [build-system] requires = ["hatchling"] diff --git a/docker-compose.yml b/docker-compose.yml index 62e1b83..fe4d7da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,14 +3,16 @@ # ============================================= # 基础部署: docker compose up -d # Agent 模式: docker compose --profile agent up -d +# 查看日志: docker compose logs -f services: # ============================================= # 核心服务 # ============================================= - + db: image: postgres:15-alpine + restart: unless-stopped volumes: - postgres_data:/var/lib/postgresql/data environment: @@ -30,6 +32,7 @@ services: backend: build: context: ./backend + restart: unless-stopped volumes: - backend_uploads:/app/uploads ports: @@ -48,6 +51,7 @@ services: frontend: build: context: ./frontend + restart: unless-stopped ports: - "3000:3000" environment: @@ -61,11 +65,12 @@ services: # Agent 审计模式服务 (可选) # 使用 --profile agent 启用 # ============================================= - + # Milvus 向量数据库 (用于 RAG 功能) milvus-etcd: image: quay.io/coreos/etcd:v3.5.5 profiles: ["agent"] + restart: unless-stopped environment: - ETCD_AUTO_COMPACTION_MODE=revision - ETCD_AUTO_COMPACTION_RETENTION=1000 @@ -85,6 +90,7 @@ services: milvus-minio: image: minio/minio:RELEASE.2023-03-20T20-16-18Z profiles: ["agent"] + restart: unless-stopped environment: MINIO_ACCESS_KEY: minioadmin MINIO_SECRET_KEY: minioadmin @@ -102,6 +108,7 @@ services: milvus: image: milvusdb/milvus:v2.4-latest profiles: ["agent"] + restart: unless-stopped command: ["milvus", "run", "standalone"] security_opt: - seccomp:unconfined @@ -129,6 +136,7 @@ services: redis: image: redis:7-alpine profiles: ["agent"] + restart: unless-stopped ports: - "6379:6379" volumes: diff --git a/frontend/src/pages/AuditRules.tsx b/frontend/src/pages/AuditRules.tsx index ffcd128..e7cd921 100644 --- a/frontend/src/pages/AuditRules.tsx +++ b/frontend/src/pages/AuditRules.tsx @@ -435,7 +435,7 @@ export default function AuditRules() { {/* Create Rule Set Dialog */} - + 新建规则集 @@ -479,7 +479,7 @@ export default function AuditRules() { {/* Edit Rule Set Dialog */} - + 编辑规则集 @@ -519,7 +519,7 @@ export default function AuditRules() { {/* Rule Edit Dialog */} - + {selectedRule ? '编辑规则' : '添加规则'} @@ -577,7 +577,7 @@ export default function AuditRules() { {/* Import Dialog */} - + 导入规则集 diff --git a/scripts/release.sh b/scripts/release.sh index baacb5d..ecc7db8 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -105,11 +105,34 @@ cd frontend npm version "$NEW_VERSION" --no-git-tag-version cd .. +# 更新后端 pyproject.toml +print_info "更新后端 pyproject.toml..." +if [ -f "backend/pyproject.toml" ]; then + sed -i.bak "s/^version = \".*\"/version = \"$NEW_VERSION\"/" backend/pyproject.toml + rm -f backend/pyproject.toml.bak +fi + +# 更新 README.md 中的版本徽章 +print_info "更新 README.md 版本徽章..." +if [ -f "README.md" ]; then + sed -i.bak "s/version-[0-9]*\.[0-9]*\.[0-9]*/version-$NEW_VERSION/" README.md + rm -f README.md.bak +fi + +# 更新 docker-compose.yml 中的版本注释 +print_info "更新 docker-compose.yml 版本注释..." +if [ -f "docker-compose.yml" ]; then + sed -i.bak "s/DeepAudit v[0-9]*\.[0-9]*\.[0-9]*/DeepAudit v$NEW_VERSION/" docker-compose.yml + rm -f docker-compose.yml.bak +fi + # 提交更改 print_info "提交版本更改..." git add frontend/package.json frontend/package-lock.json 2>/dev/null || true git add frontend/pnpm-lock.yaml 2>/dev/null || true +git add backend/pyproject.toml 2>/dev/null || true git add README.md 2>/dev/null || true +git add docker-compose.yml 2>/dev/null || true git commit -m "chore: bump version to v$NEW_VERSION" || true # 创建 tag