feat: Centralize Git tokens to system environment variables and add Gitea branch verification.

This commit is contained in:
vinland100 2026-01-05 17:12:47 +08:00
parent 8c096c00cc
commit 9ec07a6594
7 changed files with 182 additions and 81 deletions

View File

@ -290,11 +290,10 @@ async def _execute_agent_task(task_id: str):
# 获取用户配置(需要在获取项目根目录之前,以便传递 token # 获取用户配置(需要在获取项目根目录之前,以便传递 token
user_config = await _get_user_config(db, task.created_by) user_config = await _get_user_config(db, task.created_by)
# 从用户配置中提取 token和SSH密钥用于私有仓库克隆 # Git Token 始终来自系统默认(.env逻辑锁定
other_config = (user_config or {}).get('otherConfig', {}) github_token = settings.GITHUB_TOKEN
github_token = other_config.get('githubToken') or settings.GITHUB_TOKEN gitlab_token = settings.GITLAB_TOKEN
gitlab_token = other_config.get('gitlabToken') or settings.GITLAB_TOKEN gitea_token = settings.GITEA_TOKEN
gitea_token = other_config.get('giteaToken') or settings.GITEA_TOKEN
# 解密SSH私钥 # 解密SSH私钥
ssh_private_key = None ssh_private_key = None

View File

@ -24,11 +24,11 @@ SENSITIVE_LLM_FIELDS = [
'qwenApiKey', 'deepseekApiKey', 'zhipuApiKey', 'moonshotApiKey', 'qwenApiKey', 'deepseekApiKey', 'zhipuApiKey', 'moonshotApiKey',
'baiduApiKey', 'minimaxApiKey', 'doubaoApiKey' 'baiduApiKey', 'minimaxApiKey', 'doubaoApiKey'
] ]
SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken'] SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken', 'giteaToken']
def mask_api_key(key: Optional[str]) -> str: def mask_api_key(key: Optional[str]) -> str:
"""部分遮盖API Key显示前3位和后4位""" """部分遮盖API Key/Token显示前3位和后4位"""
if not key: if not key:
return "" return ""
if len(key) <= 8: if len(key) <= 8:
@ -41,6 +41,9 @@ def encrypt_config(config: dict, sensitive_fields: list) -> dict:
encrypted = config.copy() encrypted = config.copy()
for field in sensitive_fields: for field in sensitive_fields:
if field in encrypted and encrypted[field]: if field in encrypted and encrypted[field]:
# 如果已经是掩码后的值,不要加密(说明是从前端传回来的掩码值)
if "***" in str(encrypted[field]):
continue
encrypted[field] = encrypt_sensitive_data(encrypted[field]) encrypted[field] = encrypt_sensitive_data(encrypted[field])
return encrypted return encrypted
@ -50,7 +53,11 @@ def decrypt_config(config: dict, sensitive_fields: list) -> dict:
decrypted = config.copy() decrypted = config.copy()
for field in sensitive_fields: for field in sensitive_fields:
if field in decrypted and decrypted[field]: if field in decrypted and decrypted[field]:
decrypted[field] = decrypt_sensitive_data(decrypted[field]) try:
decrypted[field] = decrypt_sensitive_data(decrypted[field])
except Exception:
# 如果解密失败,保留原样
pass
return decrypted return decrypted
@ -83,6 +90,7 @@ class OtherConfigSchema(BaseModel):
"""其他配置Schema""" """其他配置Schema"""
githubToken: Optional[str] = None githubToken: Optional[str] = None
gitlabToken: Optional[str] = None gitlabToken: Optional[str] = None
giteaToken: Optional[str] = None
maxAnalyzeFiles: Optional[int] = None maxAnalyzeFiles: Optional[int] = None
llmConcurrency: Optional[int] = None llmConcurrency: Optional[int] = None
llmGapMs: Optional[int] = None llmGapMs: Optional[int] = None
@ -133,8 +141,9 @@ def get_default_config() -> dict:
"ollamaBaseUrl": settings.OLLAMA_BASE_URL or "http://localhost:11434/v1", "ollamaBaseUrl": settings.OLLAMA_BASE_URL or "http://localhost:11434/v1",
}, },
"otherConfig": { "otherConfig": {
"githubToken": settings.GITHUB_TOKEN or "", "githubToken": mask_api_key(settings.GITHUB_TOKEN),
"gitlabToken": settings.GITLAB_TOKEN or "", "gitlabToken": mask_api_key(settings.GITLAB_TOKEN),
"giteaToken": mask_api_key(settings.GITEA_TOKEN),
"maxAnalyzeFiles": settings.MAX_ANALYZE_FILES, "maxAnalyzeFiles": settings.MAX_ANALYZE_FILES,
"llmConcurrency": settings.LLM_CONCURRENCY, "llmConcurrency": settings.LLM_CONCURRENCY,
"llmGapMs": settings.LLM_GAP_MS, "llmGapMs": settings.LLM_GAP_MS,
@ -182,14 +191,15 @@ async def get_my_config(
user_llm_config = decrypt_config(user_llm_config, SENSITIVE_LLM_FIELDS) user_llm_config = decrypt_config(user_llm_config, SENSITIVE_LLM_FIELDS)
user_other_config = decrypt_config(user_other_config, SENSITIVE_OTHER_FIELDS) user_other_config = decrypt_config(user_other_config, SENSITIVE_OTHER_FIELDS)
print(f"[Config] 用户 {current_user.id} 的保存配置:")
print(f" - llmProvider: {user_llm_config.get('llmProvider')}")
print(f" - llmApiKey: {'***' + user_llm_config.get('llmApiKey', '')[-4:] if user_llm_config.get('llmApiKey') else '(空)'}")
print(f" - llmModel: {user_llm_config.get('llmModel')}")
# LLM配置始终来自系统默认.env不再允许用户覆盖 # LLM配置始终来自系统默认.env不再允许用户覆盖
merged_llm_config = default_config["llmConfig"] merged_llm_config = default_config["llmConfig"]
# Git Token 也始终来自系统默认(.env不再允许用户覆盖
merged_other_config = {**default_config["otherConfig"], **user_other_config} merged_other_config = {**default_config["otherConfig"], **user_other_config}
# 强制覆盖为默认配置中的 Token已脱敏
merged_other_config["githubToken"] = default_config["otherConfig"]["githubToken"]
merged_other_config["gitlabToken"] = default_config["otherConfig"]["gitlabToken"]
merged_other_config["giteaToken"] = default_config["otherConfig"]["giteaToken"]
return UserConfigResponse( return UserConfigResponse(
id=config.id, id=config.id,
@ -217,6 +227,14 @@ async def update_my_config(
llm_data = config_in.llmConfig.dict(exclude_none=True) if config_in.llmConfig else {} llm_data = config_in.llmConfig.dict(exclude_none=True) if config_in.llmConfig else {}
other_data = config_in.otherConfig.dict(exclude_none=True) if config_in.otherConfig else {} other_data = config_in.otherConfig.dict(exclude_none=True) if config_in.otherConfig else {}
# 如果传回来的是掩码,说明没有修改,不需要更新
if 'githubToken' in other_data and '***' in str(other_data['githubToken']):
del other_data['githubToken']
if 'gitlabToken' in other_data and '***' in str(other_data['gitlabToken']):
del other_data['gitlabToken']
if 'giteaToken' in other_data and '***' in str(other_data['giteaToken']):
del other_data['giteaToken']
# 加密敏感字段 # 加密敏感字段
llm_data_encrypted = encrypt_config(llm_data, SENSITIVE_LLM_FIELDS) llm_data_encrypted = encrypt_config(llm_data, SENSITIVE_LLM_FIELDS)
other_data_encrypted = encrypt_config(other_data, SENSITIVE_OTHER_FIELDS) other_data_encrypted = encrypt_config(other_data, SENSITIVE_OTHER_FIELDS)

View File

@ -424,15 +424,8 @@ async def get_project_files(
if config and config.other_config: if config and config.other_config:
other_config = json.loads(config.other_config) other_config = json.loads(config.other_config)
for field in SENSITIVE_OTHER_FIELDS: if 'sshPrivateKey' in other_config and other_config['sshPrivateKey']:
if field in other_config and other_config[field]: ssh_private_key = decrypt_sensitive_data(other_config['sshPrivateKey'])
decrypted_val = decrypt_sensitive_data(other_config[field])
if field == 'githubToken':
github_token = decrypted_val
elif field == 'gitlabToken':
gitlab_token = decrypted_val
elif field == 'sshPrivateKey':
ssh_private_key = decrypted_val
# 检查是否为SSH URL # 检查是否为SSH URL
is_ssh_url = GitSSHOperations.is_ssh_url(project.repository_url) is_ssh_url = GitSSHOperations.is_ssh_url(project.repository_url)
@ -716,20 +709,7 @@ async def get_project_branches(
gitea_token = settings.GITEA_TOKEN gitea_token = settings.GITEA_TOKEN
gitlab_token = settings.GITLAB_TOKEN gitlab_token = settings.GITLAB_TOKEN
SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken', 'giteaToken'] # Git Token 始终来自 .env不再从 config 中获取
if config and config.other_config:
import json
other_config = json.loads(config.other_config)
for field in SENSITIVE_OTHER_FIELDS:
if field in other_config and other_config[field]:
decrypted_val = decrypt_sensitive_data(other_config[field])
if field == 'githubToken':
github_token = decrypted_val
elif field == 'gitlabToken':
gitlab_token = decrypted_val
elif field == 'giteaToken':
gitea_token = decrypted_val
repo_type = project.repository_type or "other" repo_type = project.repository_type or "other"
@ -765,11 +745,13 @@ async def get_project_branches(
return {"branches": branches, "default_branch": default_branch} return {"branches": branches, "default_branch": default_branch}
except Exception as e: except Exception as e:
import traceback
error_msg = str(e) error_msg = str(e)
print(f"[Branch] 获取分支列表失败: {error_msg}") print(f"[Branch] 获取分支列表失败: {error_msg}")
# 返回默认分支作为后备 print(traceback.format_exc())
# 返回默认分支作为后备,并包含错误详情
return { return {
"branches": [project.default_branch or "main"], "branches": [project.default_branch or "main"],
"default_branch": project.default_branch or "main", "default_branch": project.default_branch or "main",
"error": str(e) "error": error_msg
} }

View File

@ -110,12 +110,20 @@ async def github_api(url: str, token: str = None) -> Any:
headers["Authorization"] = f"Bearer {t}" headers["Authorization"] = f"Bearer {t}"
async with httpx.AsyncClient(timeout=30) as client: async with httpx.AsyncClient(timeout=30) as client:
response = await client.get(url, headers=headers) try:
if response.status_code == 403: response = await client.get(url, headers=headers)
raise Exception("GitHub API 403请配置 GITHUB_TOKEN 或确认仓库权限/频率限制") if response.status_code == 403:
if response.status_code != 200: raise Exception("GitHub API 403请配置 GITHUB_TOKEN 或确认仓库权限/频率限制")
raise Exception(f"GitHub API {response.status_code}: {url}") if response.status_code != 200:
return response.json() raise Exception(f"GitHub API {response.status_code}: {url}")
data = response.json()
if not isinstance(data, (list, dict)):
print(f"[API] 警告: GitHub API 返回了非预期的格式: {type(data)}")
return data
except Exception as e:
print(f"[API] GitHub API 调用失败: {url}, 错误: {e}")
raise
@ -127,14 +135,22 @@ async def gitea_api(url: str, token: str = None) -> Any:
headers["Authorization"] = f"token {t}" headers["Authorization"] = f"token {t}"
async with httpx.AsyncClient(timeout=30) as client: async with httpx.AsyncClient(timeout=30) as client:
response = await client.get(url, headers=headers) try:
if response.status_code == 401: response = await client.get(url, headers=headers)
raise Exception("Gitea API 401请配置 GITEA_TOKEN 或确认仓库权限") if response.status_code == 401:
if response.status_code == 403: raise Exception("Gitea API 401请配置 GITEA_TOKEN 或确认仓库权限")
raise Exception("Gitea API 403请确认仓库权限/频率限制") if response.status_code == 403:
if response.status_code != 200: raise Exception("Gitea API 403请确认仓库权限/频率限制")
raise Exception(f"Gitea API {response.status_code}: {url}") if response.status_code != 200:
return response.json() raise Exception(f"Gitea API {response.status_code}: {url}")
data = response.json()
if not isinstance(data, (list, dict)):
print(f"[API] 警告: Gitea API 返回了非预期的格式: {type(data)}")
return data
except Exception as e:
print(f"[API] Gitea API 调用失败: {url}, 错误: {e}")
raise
async def gitlab_api(url: str, token: str = None) -> Any: async def gitlab_api(url: str, token: str = None) -> Any:
@ -145,14 +161,22 @@ async def gitlab_api(url: str, token: str = None) -> Any:
headers["PRIVATE-TOKEN"] = t headers["PRIVATE-TOKEN"] = t
async with httpx.AsyncClient(timeout=30) as client: async with httpx.AsyncClient(timeout=30) as client:
response = await client.get(url, headers=headers) try:
if response.status_code == 401: response = await client.get(url, headers=headers)
raise Exception("GitLab API 401请配置 GITLAB_TOKEN 或确认仓库权限") if response.status_code == 401:
if response.status_code == 403: raise Exception("GitLab API 401请配置 GITLAB_TOKEN 或确认仓库权限")
raise Exception("GitLab API 403请确认仓库权限/频率限制") if response.status_code == 403:
if response.status_code != 200: raise Exception("GitLab API 403请确认仓库权限/频率限制")
raise Exception(f"GitLab API {response.status_code}: {url}") if response.status_code != 200:
return response.json() raise Exception(f"GitLab API {response.status_code}: {url}")
data = response.json()
if not isinstance(data, (list, dict)):
print(f"[API] 警告: GitLab API 返回了非预期的格式: {type(data)}")
return data
except Exception as e:
print(f"[API] GitLab API 调用失败: {url}, 错误: {e}")
raise
async def fetch_file_content(url: str, headers: Dict[str, str] = None) -> Optional[str]: async def fetch_file_content(url: str, headers: Dict[str, str] = None) -> Optional[str]:
@ -175,7 +199,11 @@ async def get_github_branches(repo_url: str, token: str = None) -> List[str]:
branches_url = f"https://api.github.com/repos/{owner}/{repo}/branches?per_page=100" branches_url = f"https://api.github.com/repos/{owner}/{repo}/branches?per_page=100"
branches_data = await github_api(branches_url, token) branches_data = await github_api(branches_url, token)
return [b["name"] for b in branches_data] if not isinstance(branches_data, list):
print(f"[Branch] 警告: 获取 GitHub 分支列表返回非列表数据: {branches_data}")
return []
return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b]
@ -190,7 +218,11 @@ async def get_gitea_branches(repo_url: str, token: str = None) -> List[str]:
branches_url = f"{base_url}/repos/{owner}/{repo}/branches" branches_url = f"{base_url}/repos/{owner}/{repo}/branches"
branches_data = await gitea_api(branches_url, token) branches_data = await gitea_api(branches_url, token)
return [b["name"] for b in branches_data] if not isinstance(branches_data, list):
print(f"[Branch] 警告: 获取 Gitea 分支列表返回非列表数据: {branches_data}")
return []
return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b]
async def get_gitlab_branches(repo_url: str, token: str = None) -> List[str]: async def get_gitlab_branches(repo_url: str, token: str = None) -> List[str]:
@ -211,7 +243,11 @@ async def get_gitlab_branches(repo_url: str, token: str = None) -> List[str]:
branches_url = f"{base_url}/projects/{project_path}/repository/branches?per_page=100" branches_url = f"{base_url}/projects/{project_path}/repository/branches?per_page=100"
branches_data = await gitlab_api(branches_url, extracted_token) branches_data = await gitlab_api(branches_url, extracted_token)
return [b["name"] for b in branches_data] if not isinstance(branches_data, list):
print(f"[Branch] 警告: 获取 GitLab 分支列表返回非列表数据: {branches_data}")
return []
return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b]
async def get_github_files(repo_url: str, branch: str, token: str = None, exclude_patterns: List[str] = None) -> List[Dict[str, str]]: async def get_github_files(repo_url: str, branch: str, token: str = None, exclude_patterns: List[str] = None) -> List[Dict[str, str]]:
@ -348,11 +384,10 @@ async def scan_repo_task(task_id: str, db_session_factory, user_config: dict = N
print(f"📋 排除模式: {task_exclude_patterns}") print(f"📋 排除模式: {task_exclude_patterns}")
# 3. 获取文件列表 # 3. 获取文件列表
# 从用户配置中读取 GitHub/GitLab Token优先使用用户配置然后使用系统配置 # Git Token 始终来自系统默认(.env逻辑锁定
user_other_config = (user_config or {}).get('otherConfig', {}) github_token = settings.GITHUB_TOKEN
github_token = user_other_config.get('githubToken') or settings.GITHUB_TOKEN gitlab_token = settings.GITLAB_TOKEN
gitlab_token = user_other_config.get('gitlabToken') or settings.GITLAB_TOKEN gitea_token = settings.GITEA_TOKEN
gitea_token = user_other_config.get('giteaToken') or settings.GITEA_TOKEN

View File

@ -114,10 +114,6 @@ export default function AgentModeSelector({
className="sr-only" className="sr-only"
/> />
{/* 推荐标签 */}
<div className="absolute -top-2 -right-2 px-2 py-0.5 bg-violet-600 text-white text-xs font-bold uppercase font-mono rounded shadow-[0_0_10px_rgba(139,92,246,0.5)]">
</div>
<div className="flex items-center gap-2 mb-2"> <div className="flex items-center gap-2 mb-2">
<div className={cn( <div className={cn(

View File

@ -654,13 +654,17 @@ export function SystemConfig() {
<TabsContent value="git" className="space-y-6"> <TabsContent value="git" className="space-y-6">
<div className="cyber-card p-6 space-y-6"> <div className="cyber-card p-6 space-y-6">
<div className="space-y-2"> <div className="space-y-2">
<Label className="text-xs font-bold text-muted-foreground uppercase">GitHub Token ()</Label> <Label className="text-xs font-bold text-muted-foreground uppercase flex items-center justify-between">
<span>GitHub Token ()</span>
<span className="text-[10px] text-amber-500/80 normal-case font-normal border border-amber-500/30 px-1 rounded">.env ()</span>
</Label>
<Input <Input
type="password" type="text"
value={config.githubToken} value={config.githubToken}
onChange={(e) => updateConfig('githubToken', e.target.value)} onChange={(e) => updateConfig('githubToken', e.target.value)}
placeholder="ghp_xxxxxxxxxxxx" placeholder="ghp_xxxxxxxxxxxx"
className="h-10 cyber-input" className="h-10 cyber-input"
disabled
/> />
<p className="text-xs text-muted-foreground"> <p className="text-xs text-muted-foreground">
访:{' '} 访:{' '}
@ -670,13 +674,17 @@ export function SystemConfig() {
</p> </p>
</div> </div>
<div className="space-y-2"> <div className="space-y-2">
<Label className="text-xs font-bold text-muted-foreground uppercase">GitLab Token ()</Label> <Label className="text-xs font-bold text-muted-foreground uppercase flex items-center justify-between">
<span>GitLab Token ()</span>
<span className="text-[10px] text-amber-500/80 normal-case font-normal border border-amber-500/30 px-1 rounded">.env ()</span>
</Label>
<Input <Input
type="password" type="text"
value={config.gitlabToken} value={config.gitlabToken}
onChange={(e) => updateConfig('gitlabToken', e.target.value)} onChange={(e) => updateConfig('gitlabToken', e.target.value)}
placeholder="glpat-xxxxxxxxxxxx" placeholder="glpat-xxxxxxxxxxxx"
className="h-10 cyber-input" className="h-10 cyber-input"
disabled
/> />
<p className="text-xs text-muted-foreground"> <p className="text-xs text-muted-foreground">
访:{' '} 访:{' '}
@ -686,13 +694,17 @@ export function SystemConfig() {
</p> </p>
</div> </div>
<div className="space-y-2"> <div className="space-y-2">
<Label className="text-xs font-bold text-muted-foreground uppercase">Gitea Token ()</Label> <Label className="text-xs font-bold text-muted-foreground uppercase flex items-center justify-between">
<span>Gitea Token ()</span>
<span className="text-[10px] text-amber-500/80 normal-case font-normal border border-amber-500/30 px-1 rounded">.env ()</span>
</Label>
<Input <Input
type="password" type="text"
value={config.giteaToken} value={config.giteaToken}
onChange={(e) => updateConfig('giteaToken', e.target.value)} onChange={(e) => updateConfig('giteaToken', e.target.value)}
placeholder="sha1_xxxxxxxxxxxx" placeholder="sha1_xxxxxxxxxxxx"
className="h-10 cyber-input" className="h-10 cyber-input"
disabled
/> />
<p className="text-xs text-muted-foreground"> <p className="text-xs text-muted-foreground">
访 Gitea :{' '} 访 Gitea :{' '}
@ -706,8 +718,9 @@ export function SystemConfig() {
<Info className="w-4 h-4 text-sky-400" /> <Info className="w-4 h-4 text-sky-400" />
</p> </p>
<p className="text-muted-foreground"> Git .env </p>
<p className="text-muted-foreground"> .env </p>
<p className="text-muted-foreground"> Token</p> <p className="text-muted-foreground"> Token</p>
<p className="text-muted-foreground"> Token</p>
</div> </div>
</div> </div>

View File

@ -0,0 +1,58 @@
import asyncio
import sys
import os
# Add backend to sys.path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'backend')))
from unittest.mock import AsyncMock, patch
from app.services.scanner import get_gitea_branches
async def test_gitea_branches():
print("Testing Gitea branch fetching with various mocked responses...")
repo_url = "http://gitea.example.com/owner/repo"
token = "test-token"
test_cases = [
{
"name": "Normal list response",
"mock_response": [{"name": "main"}, {"name": "develop"}],
"expected_count": 2
},
{
"name": "Null (None) response",
"mock_response": None,
"expected_count": 0
},
{
"name": "Empty list response",
"mock_response": [],
"expected_count": 0
},
{
"name": "Dict response (unexpected)",
"mock_response": {"message": "Not found"},
"expected_count": 0
},
{
"name": "List with invalid items",
"mock_response": [{"name": "main"}, "invalid", {"other": "data"}],
"expected_count": 1
}
]
for case in test_cases:
print(f"\nCase: {case['name']}")
with patch('app.services.scanner.gitea_api', new_callable=AsyncMock) as mock_api:
mock_api.return_value = case['mock_response']
try:
branches = await get_gitea_branches(repo_url, token)
print(f"Result: {branches}")
assert len(branches) == case['expected_count']
print("✅ Pass")
except Exception as e:
print(f"❌ Fail with exception: {e}")
if __name__ == "__main__":
asyncio.run(test_gitea_branches())