From 9ec07a65942a4cea58b0edf81f4931c12fe22473 Mon Sep 17 00:00:00 2001 From: vinland100 Date: Mon, 5 Jan 2026 17:12:47 +0800 Subject: [PATCH] feat: Centralize Git tokens to system environment variables and add Gitea branch verification. --- backend/app/api/v1/endpoints/agent_tasks.py | 9 +- backend/app/api/v1/endpoints/config.py | 38 ++++++-- backend/app/api/v1/endpoints/projects.py | 32 ++----- backend/app/services/scanner.py | 95 +++++++++++++------ .../components/agent/AgentModeSelector.tsx | 4 - .../src/components/system/SystemConfig.tsx | 27 ++++-- tests/verify_gitea_branches.py | 58 +++++++++++ 7 files changed, 182 insertions(+), 81 deletions(-) create mode 100644 tests/verify_gitea_branches.py diff --git a/backend/app/api/v1/endpoints/agent_tasks.py b/backend/app/api/v1/endpoints/agent_tasks.py index b1f8341..3a6b465 100644 --- a/backend/app/api/v1/endpoints/agent_tasks.py +++ b/backend/app/api/v1/endpoints/agent_tasks.py @@ -290,11 +290,10 @@ async def _execute_agent_task(task_id: str): # 获取用户配置（需要在获取项目根目录之前，以便传递 token） user_config = await _get_user_config(db, task.created_by) - # 从用户配置中提取 token和SSH密钥（用于私有仓库克隆） - other_config = (user_config or {}).get('otherConfig', {}) - github_token = other_config.get('githubToken') or settings.GITHUB_TOKEN - gitlab_token = other_config.get('gitlabToken') or settings.GITLAB_TOKEN - gitea_token = other_config.get('giteaToken') or settings.GITEA_TOKEN + # Git Token 始终来自系统默认（.env），逻辑锁定 + github_token = settings.GITHUB_TOKEN + gitlab_token = settings.GITLAB_TOKEN + gitea_token = settings.GITEA_TOKEN # 解密SSH私钥 ssh_private_key = None diff --git a/backend/app/api/v1/endpoints/config.py b/backend/app/api/v1/endpoints/config.py index 35ece2e..a7fdcd4 100644 --- a/backend/app/api/v1/endpoints/config.py +++ b/backend/app/api/v1/endpoints/config.py @@ -24,11 +24,11 @@ SENSITIVE_LLM_FIELDS = [ 'qwenApiKey', 'deepseekApiKey', 'zhipuApiKey', 'moonshotApiKey', 'baiduApiKey', 'minimaxApiKey', 'doubaoApiKey' ] -SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken'] +SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken', 'giteaToken'] def mask_api_key(key: Optional[str]) -> str: - """部分遮盖API Key，显示前3位和后4位""" + """部分遮盖API Key/Token，显示前3位和后4位""" if not key: return "" if len(key) <= 8: @@ -41,6 +41,9 @@ def encrypt_config(config: dict, sensitive_fields: list) -> dict: encrypted = config.copy() for field in sensitive_fields: if field in encrypted and encrypted[field]: + # 如果已经是掩码后的值，不要加密（说明是从前端传回来的掩码值） + if "***" in str(encrypted[field]): + continue encrypted[field] = encrypt_sensitive_data(encrypted[field]) return encrypted @@ -50,7 +53,11 @@ def decrypt_config(config: dict, sensitive_fields: list) -> dict: decrypted = config.copy() for field in sensitive_fields: if field in decrypted and decrypted[field]: - decrypted[field] = decrypt_sensitive_data(decrypted[field]) + try: + decrypted[field] = decrypt_sensitive_data(decrypted[field]) + except Exception: + # 如果解密失败，保留原样 + pass return decrypted @@ -83,6 +90,7 @@ class OtherConfigSchema(BaseModel): """其他配置Schema""" githubToken: Optional[str] = None gitlabToken: Optional[str] = None + giteaToken: Optional[str] = None maxAnalyzeFiles: Optional[int] = None llmConcurrency: Optional[int] = None llmGapMs: Optional[int] = None @@ -133,8 +141,9 @@ def get_default_config() -> dict: "ollamaBaseUrl": settings.OLLAMA_BASE_URL or "http://localhost:11434/v1", }, "otherConfig": { - "githubToken": settings.GITHUB_TOKEN or "", - "gitlabToken": settings.GITLAB_TOKEN or "", + "githubToken": mask_api_key(settings.GITHUB_TOKEN), + "gitlabToken": mask_api_key(settings.GITLAB_TOKEN), + "giteaToken": mask_api_key(settings.GITEA_TOKEN), "maxAnalyzeFiles": settings.MAX_ANALYZE_FILES, "llmConcurrency": settings.LLM_CONCURRENCY, "llmGapMs": settings.LLM_GAP_MS, @@ -182,14 +191,15 @@ async def get_my_config( user_llm_config = decrypt_config(user_llm_config, SENSITIVE_LLM_FIELDS) user_other_config = decrypt_config(user_other_config, SENSITIVE_OTHER_FIELDS) - print(f"[Config] 用户 {current_user.id} 的保存配置:") - print(f" - llmProvider: {user_llm_config.get('llmProvider')}") - print(f" - llmApiKey: {'***' + user_llm_config.get('llmApiKey', '')[-4:] if user_llm_config.get('llmApiKey') else '(空)'}") - print(f" - llmModel: {user_llm_config.get('llmModel')}") - # LLM配置始终来自系统默认（.env），不再允许用户覆盖 merged_llm_config = default_config["llmConfig"] + + # Git Token 也始终来自系统默认（.env），不再允许用户覆盖 merged_other_config = {**default_config["otherConfig"], **user_other_config} + # 强制覆盖为默认配置中的 Token（已脱敏） + merged_other_config["githubToken"] = default_config["otherConfig"]["githubToken"] + merged_other_config["gitlabToken"] = default_config["otherConfig"]["gitlabToken"] + merged_other_config["giteaToken"] = default_config["otherConfig"]["giteaToken"] return UserConfigResponse( id=config.id, @@ -217,6 +227,14 @@ async def update_my_config( llm_data = config_in.llmConfig.dict(exclude_none=True) if config_in.llmConfig else {} other_data = config_in.otherConfig.dict(exclude_none=True) if config_in.otherConfig else {} + # 如果传回来的是掩码，说明没有修改，不需要更新 + if 'githubToken' in other_data and '***' in str(other_data['githubToken']): + del other_data['githubToken'] + if 'gitlabToken' in other_data and '***' in str(other_data['gitlabToken']): + del other_data['gitlabToken'] + if 'giteaToken' in other_data and '***' in str(other_data['giteaToken']): + del other_data['giteaToken'] + # 加密敏感字段 llm_data_encrypted = encrypt_config(llm_data, SENSITIVE_LLM_FIELDS) other_data_encrypted = encrypt_config(other_data, SENSITIVE_OTHER_FIELDS) diff --git a/backend/app/api/v1/endpoints/projects.py b/backend/app/api/v1/endpoints/projects.py index e26143a..e4c8a59 100644 --- a/backend/app/api/v1/endpoints/projects.py +++ b/backend/app/api/v1/endpoints/projects.py @@ -424,15 +424,8 @@ async def get_project_files( if config and config.other_config: other_config = json.loads(config.other_config) - for field in SENSITIVE_OTHER_FIELDS: - if field in other_config and other_config[field]: - decrypted_val = decrypt_sensitive_data(other_config[field]) - if field == 'githubToken': - github_token = decrypted_val - elif field == 'gitlabToken': - gitlab_token = decrypted_val - elif field == 'sshPrivateKey': - ssh_private_key = decrypted_val + if 'sshPrivateKey' in other_config and other_config['sshPrivateKey']: + ssh_private_key = decrypt_sensitive_data(other_config['sshPrivateKey']) # 检查是否为SSH URL is_ssh_url = GitSSHOperations.is_ssh_url(project.repository_url) @@ -716,20 +709,7 @@ async def get_project_branches( gitea_token = settings.GITEA_TOKEN gitlab_token = settings.GITLAB_TOKEN - SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken', 'giteaToken'] - - if config and config.other_config: - import json - other_config = json.loads(config.other_config) - for field in SENSITIVE_OTHER_FIELDS: - if field in other_config and other_config[field]: - decrypted_val = decrypt_sensitive_data(other_config[field]) - if field == 'githubToken': - github_token = decrypted_val - elif field == 'gitlabToken': - gitlab_token = decrypted_val - elif field == 'giteaToken': - gitea_token = decrypted_val + # Git Token 始终来自 .env，不再从 config 中获取 repo_type = project.repository_type or "other" @@ -765,11 +745,13 @@ async def get_project_branches( return {"branches": branches, "default_branch": default_branch} except Exception as e: + import traceback error_msg = str(e) print(f"[Branch] 获取分支列表失败: {error_msg}") - # 返回默认分支作为后备 + print(traceback.format_exc()) + # 返回默认分支作为后备，并包含错误详情 return { "branches": [project.default_branch or "main"], "default_branch": project.default_branch or "main", - "error": str(e) + "error": error_msg } diff --git a/backend/app/services/scanner.py b/backend/app/services/scanner.py index 4bf2892..3fe6d99 100644 --- a/backend/app/services/scanner.py +++ b/backend/app/services/scanner.py @@ -110,12 +110,20 @@ async def github_api(url: str, token: str = None) -> Any: headers["Authorization"] = f"Bearer {t}" async with httpx.AsyncClient(timeout=30) as client: - response = await client.get(url, headers=headers) - if response.status_code == 403: - raise Exception("GitHub API 403：请配置 GITHUB_TOKEN 或确认仓库权限/频率限制") - if response.status_code != 200: - raise Exception(f"GitHub API {response.status_code}: {url}") - return response.json() + try: + response = await client.get(url, headers=headers) + if response.status_code == 403: + raise Exception("GitHub API 403：请配置 GITHUB_TOKEN 或确认仓库权限/频率限制") + if response.status_code != 200: + raise Exception(f"GitHub API {response.status_code}: {url}") + + data = response.json() + if not isinstance(data, (list, dict)): + print(f"[API] 警告: GitHub API 返回了非预期的格式: {type(data)}") + return data + except Exception as e: + print(f"[API] GitHub API 调用失败: {url}, 错误: {e}") + raise @@ -127,14 +135,22 @@ async def gitea_api(url: str, token: str = None) -> Any: headers["Authorization"] = f"token {t}" async with httpx.AsyncClient(timeout=30) as client: - response = await client.get(url, headers=headers) - if response.status_code == 401: - raise Exception("Gitea API 401：请配置 GITEA_TOKEN 或确认仓库权限") - if response.status_code == 403: - raise Exception("Gitea API 403：请确认仓库权限/频率限制") - if response.status_code != 200: - raise Exception(f"Gitea API {response.status_code}: {url}") - return response.json() + try: + response = await client.get(url, headers=headers) + if response.status_code == 401: + raise Exception("Gitea API 401：请配置 GITEA_TOKEN 或确认仓库权限") + if response.status_code == 403: + raise Exception("Gitea API 403：请确认仓库权限/频率限制") + if response.status_code != 200: + raise Exception(f"Gitea API {response.status_code}: {url}") + + data = response.json() + if not isinstance(data, (list, dict)): + print(f"[API] 警告: Gitea API 返回了非预期的格式: {type(data)}") + return data + except Exception as e: + print(f"[API] Gitea API 调用失败: {url}, 错误: {e}") + raise async def gitlab_api(url: str, token: str = None) -> Any: @@ -145,14 +161,22 @@ async def gitlab_api(url: str, token: str = None) -> Any: headers["PRIVATE-TOKEN"] = t async with httpx.AsyncClient(timeout=30) as client: - response = await client.get(url, headers=headers) - if response.status_code == 401: - raise Exception("GitLab API 401：请配置 GITLAB_TOKEN 或确认仓库权限") - if response.status_code == 403: - raise Exception("GitLab API 403：请确认仓库权限/频率限制") - if response.status_code != 200: - raise Exception(f"GitLab API {response.status_code}: {url}") - return response.json() + try: + response = await client.get(url, headers=headers) + if response.status_code == 401: + raise Exception("GitLab API 401：请配置 GITLAB_TOKEN 或确认仓库权限") + if response.status_code == 403: + raise Exception("GitLab API 403：请确认仓库权限/频率限制") + if response.status_code != 200: + raise Exception(f"GitLab API {response.status_code}: {url}") + + data = response.json() + if not isinstance(data, (list, dict)): + print(f"[API] 警告: GitLab API 返回了非预期的格式: {type(data)}") + return data + except Exception as e: + print(f"[API] GitLab API 调用失败: {url}, 错误: {e}") + raise async def fetch_file_content(url: str, headers: Dict[str, str] = None) -> Optional[str]: @@ -175,7 +199,11 @@ async def get_github_branches(repo_url: str, token: str = None) -> List[str]: branches_url = f"https://api.github.com/repos/{owner}/{repo}/branches?per_page=100" branches_data = await github_api(branches_url, token) - return [b["name"] for b in branches_data] + if not isinstance(branches_data, list): + print(f"[Branch] 警告: 获取 GitHub 分支列表返回非列表数据: {branches_data}") + return [] + + return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b] @@ -190,7 +218,11 @@ async def get_gitea_branches(repo_url: str, token: str = None) -> List[str]: branches_url = f"{base_url}/repos/{owner}/{repo}/branches" branches_data = await gitea_api(branches_url, token) - return [b["name"] for b in branches_data] + if not isinstance(branches_data, list): + print(f"[Branch] 警告: 获取 Gitea 分支列表返回非列表数据: {branches_data}") + return [] + + return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b] async def get_gitlab_branches(repo_url: str, token: str = None) -> List[str]: @@ -211,7 +243,11 @@ async def get_gitlab_branches(repo_url: str, token: str = None) -> List[str]: branches_url = f"{base_url}/projects/{project_path}/repository/branches?per_page=100" branches_data = await gitlab_api(branches_url, extracted_token) - return [b["name"] for b in branches_data] + if not isinstance(branches_data, list): + print(f"[Branch] 警告: 获取 GitLab 分支列表返回非列表数据: {branches_data}") + return [] + + return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b] async def get_github_files(repo_url: str, branch: str, token: str = None, exclude_patterns: List[str] = None) -> List[Dict[str, str]]: @@ -348,11 +384,10 @@ async def scan_repo_task(task_id: str, db_session_factory, user_config: dict = N print(f"📋 排除模式: {task_exclude_patterns}") # 3. 获取文件列表 - # 从用户配置中读取 GitHub/GitLab Token（优先使用用户配置，然后使用系统配置） - user_other_config = (user_config or {}).get('otherConfig', {}) - github_token = user_other_config.get('githubToken') or settings.GITHUB_TOKEN - gitlab_token = user_other_config.get('gitlabToken') or settings.GITLAB_TOKEN - gitea_token = user_other_config.get('giteaToken') or settings.GITEA_TOKEN + # Git Token 始终来自系统默认（.env），逻辑锁定 + github_token = settings.GITHUB_TOKEN + gitlab_token = settings.GITLAB_TOKEN + gitea_token = settings.GITEA_TOKEN diff --git a/frontend/src/components/agent/AgentModeSelector.tsx b/frontend/src/components/agent/AgentModeSelector.tsx index ef04bbd..aab8f8d 100644 --- a/frontend/src/components/agent/AgentModeSelector.tsx +++ b/frontend/src/components/agent/AgentModeSelector.tsx @@ -114,10 +114,6 @@ export default function AgentModeSelector({ className="sr-only" /> - {/* 推荐标签 */} -
- 推荐 -
- + updateConfig('githubToken', e.target.value)} placeholder="ghp_xxxxxxxxxxxx" className="h-10 cyber-input" + disabled />

用于访问私有仓库。获取:{' '} @@ -670,13 +674,17 @@ export function SystemConfig() {

- + updateConfig('gitlabToken', e.target.value)} placeholder="glpat-xxxxxxxxxxxx" className="h-10 cyber-input" + disabled />

用于访问私有仓库。获取:{' '} @@ -686,13 +694,17 @@ export function SystemConfig() {

- + updateConfig('giteaToken', e.target.value)} placeholder="sha1_xxxxxxxxxxxx" className="h-10 cyber-input" + disabled />

用于访问 Gitea 私有仓库。获取:{' '} @@ -706,8 +718,9 @@ export function SystemConfig() { 提示

+

• Git 分布式配置已锁定至后端 .env 文件

+

• 如果需要修改，请编辑后端的 .env 环境文件并重启服务

• 公开仓库无需配置 Token

-

• 私有仓库需要配置对应平台的 Token

diff --git a/tests/verify_gitea_branches.py b/tests/verify_gitea_branches.py new file mode 100644 index 0000000..3f58301 --- /dev/null +++ b/tests/verify_gitea_branches.py @@ -0,0 +1,58 @@ +import asyncio +import sys +import os + +# Add backend to sys.path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'backend'))) + +from unittest.mock import AsyncMock, patch +from app.services.scanner import get_gitea_branches + +async def test_gitea_branches(): + print("Testing Gitea branch fetching with various mocked responses...") + + repo_url = "http://gitea.example.com/owner/repo" + token = "test-token" + + test_cases = [ + { + "name": "Normal list response", + "mock_response": [{"name": "main"}, {"name": "develop"}], + "expected_count": 2 + }, + { + "name": "Null (None) response", + "mock_response": None, + "expected_count": 0 + }, + { + "name": "Empty list response", + "mock_response": [], + "expected_count": 0 + }, + { + "name": "Dict response (unexpected)", + "mock_response": {"message": "Not found"}, + "expected_count": 0 + }, + { + "name": "List with invalid items", + "mock_response": [{"name": "main"}, "invalid", {"other": "data"}], + "expected_count": 1 + } + ] + + for case in test_cases: + print(f"\nCase: {case['name']}") + with patch('app.services.scanner.gitea_api', new_callable=AsyncMock) as mock_api: + mock_api.return_value = case['mock_response'] + try: + branches = await get_gitea_branches(repo_url, token) + print(f"Result: {branches}") + assert len(branches) == case['expected_count'] + print("✅ Pass") + except Exception as e: + print(f"❌ Fail with exception: {e}") + +if __name__ == "__main__": + asyncio.run(test_gitea_branches())