feat: Centralize Git tokens to system environment variables and add Gitea branch verification.
This commit is contained in:
vinland100
parent
8c096c00cc
commit
9ec07a6594
|
|
@ -290,11 +290,10 @@ async def _execute_agent_task(task_id: str):
|
|||
# 获取用户配置（需要在获取项目根目录之前，以便传递 token）
|
||||
user_config = await _get_user_config(db, task.created_by)
|
||||
|
||||
# 从用户配置中提取 token和SSH密钥（用于私有仓库克隆）
|
||||
other_config = (user_config or {}).get('otherConfig', {})
|
||||
github_token = other_config.get('githubToken') or settings.GITHUB_TOKEN
|
||||
gitlab_token = other_config.get('gitlabToken') or settings.GITLAB_TOKEN
|
||||
gitea_token = other_config.get('giteaToken') or settings.GITEA_TOKEN
|
||||
# Git Token 始终来自系统默认（.env），逻辑锁定
|
||||
github_token = settings.GITHUB_TOKEN
|
||||
gitlab_token = settings.GITLAB_TOKEN
|
||||
gitea_token = settings.GITEA_TOKEN
|
||||
|
||||
# 解密SSH私钥
|
||||
ssh_private_key = None
|
||||
|
|
|
|||
|
|
@ -24,11 +24,11 @@ SENSITIVE_LLM_FIELDS = [
|
|||
'qwenApiKey', 'deepseekApiKey', 'zhipuApiKey', 'moonshotApiKey',
|
||||
'baiduApiKey', 'minimaxApiKey', 'doubaoApiKey'
|
||||
]
|
||||
SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken']
|
||||
SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken', 'giteaToken']
|
||||
|
||||
|
||||
def mask_api_key(key: Optional[str]) -> str:
|
||||
"""部分遮盖API Key，显示前3位和后4位"""
|
||||
"""部分遮盖API Key/Token，显示前3位和后4位"""
|
||||
if not key:
|
||||
return ""
|
||||
if len(key) <= 8:
|
||||
|
|
@ -41,6 +41,9 @@ def encrypt_config(config: dict, sensitive_fields: list) -> dict:
|
|||
encrypted = config.copy()
|
||||
for field in sensitive_fields:
|
||||
if field in encrypted and encrypted[field]:
|
||||
# 如果已经是掩码后的值，不要加密（说明是从前端传回来的掩码值）
|
||||
if "***" in str(encrypted[field]):
|
||||
continue
|
||||
encrypted[field] = encrypt_sensitive_data(encrypted[field])
|
||||
return encrypted
|
||||
|
||||
|
|
@ -50,7 +53,11 @@ def decrypt_config(config: dict, sensitive_fields: list) -> dict:
|
|||
decrypted = config.copy()
|
||||
for field in sensitive_fields:
|
||||
if field in decrypted and decrypted[field]:
|
||||
try:
|
||||
decrypted[field] = decrypt_sensitive_data(decrypted[field])
|
||||
except Exception:
|
||||
# 如果解密失败，保留原样
|
||||
pass
|
||||
return decrypted
|
||||
|
||||
|
||||
|
|
@ -83,6 +90,7 @@ class OtherConfigSchema(BaseModel):
|
|||
"""其他配置Schema"""
|
||||
githubToken: Optional[str] = None
|
||||
gitlabToken: Optional[str] = None
|
||||
giteaToken: Optional[str] = None
|
||||
maxAnalyzeFiles: Optional[int] = None
|
||||
llmConcurrency: Optional[int] = None
|
||||
llmGapMs: Optional[int] = None
|
||||
|
|
@ -133,8 +141,9 @@ def get_default_config() -> dict:
|
|||
"ollamaBaseUrl": settings.OLLAMA_BASE_URL or "http://localhost:11434/v1",
|
||||
},
|
||||
"otherConfig": {
|
||||
"githubToken": settings.GITHUB_TOKEN or "",
|
||||
"gitlabToken": settings.GITLAB_TOKEN or "",
|
||||
"githubToken": mask_api_key(settings.GITHUB_TOKEN),
|
||||
"gitlabToken": mask_api_key(settings.GITLAB_TOKEN),
|
||||
"giteaToken": mask_api_key(settings.GITEA_TOKEN),
|
||||
"maxAnalyzeFiles": settings.MAX_ANALYZE_FILES,
|
||||
"llmConcurrency": settings.LLM_CONCURRENCY,
|
||||
"llmGapMs": settings.LLM_GAP_MS,
|
||||
|
|
@ -182,14 +191,15 @@ async def get_my_config(
|
|||
user_llm_config = decrypt_config(user_llm_config, SENSITIVE_LLM_FIELDS)
|
||||
user_other_config = decrypt_config(user_other_config, SENSITIVE_OTHER_FIELDS)
|
||||
|
||||
print(f"[Config] 用户 {current_user.id} 的保存配置:")
|
||||
print(f" - llmProvider: {user_llm_config.get('llmProvider')}")
|
||||
print(f" - llmApiKey: {'***' + user_llm_config.get('llmApiKey', '')[-4:] if user_llm_config.get('llmApiKey') else '(空)'}")
|
||||
print(f" - llmModel: {user_llm_config.get('llmModel')}")
|
||||
|
||||
# LLM配置始终来自系统默认（.env），不再允许用户覆盖
|
||||
merged_llm_config = default_config["llmConfig"]
|
||||
|
||||
# Git Token 也始终来自系统默认（.env），不再允许用户覆盖
|
||||
merged_other_config = {**default_config["otherConfig"], **user_other_config}
|
||||
# 强制覆盖为默认配置中的 Token（已脱敏）
|
||||
merged_other_config["githubToken"] = default_config["otherConfig"]["githubToken"]
|
||||
merged_other_config["gitlabToken"] = default_config["otherConfig"]["gitlabToken"]
|
||||
merged_other_config["giteaToken"] = default_config["otherConfig"]["giteaToken"]
|
||||
|
||||
return UserConfigResponse(
|
||||
id=config.id,
|
||||
|
|
@ -217,6 +227,14 @@ async def update_my_config(
|
|||
llm_data = config_in.llmConfig.dict(exclude_none=True) if config_in.llmConfig else {}
|
||||
other_data = config_in.otherConfig.dict(exclude_none=True) if config_in.otherConfig else {}
|
||||
|
||||
# 如果传回来的是掩码，说明没有修改，不需要更新
|
||||
if 'githubToken' in other_data and '***' in str(other_data['githubToken']):
|
||||
del other_data['githubToken']
|
||||
if 'gitlabToken' in other_data and '***' in str(other_data['gitlabToken']):
|
||||
del other_data['gitlabToken']
|
||||
if 'giteaToken' in other_data and '***' in str(other_data['giteaToken']):
|
||||
del other_data['giteaToken']
|
||||
|
||||
# 加密敏感字段
|
||||
llm_data_encrypted = encrypt_config(llm_data, SENSITIVE_LLM_FIELDS)
|
||||
other_data_encrypted = encrypt_config(other_data, SENSITIVE_OTHER_FIELDS)
|
||||
|
|
|
|||
|
|
@ -424,15 +424,8 @@ async def get_project_files(
|
|||
|
||||
if config and config.other_config:
|
||||
other_config = json.loads(config.other_config)
|
||||
for field in SENSITIVE_OTHER_FIELDS:
|
||||
if field in other_config and other_config[field]:
|
||||
decrypted_val = decrypt_sensitive_data(other_config[field])
|
||||
if field == 'githubToken':
|
||||
github_token = decrypted_val
|
||||
elif field == 'gitlabToken':
|
||||
gitlab_token = decrypted_val
|
||||
elif field == 'sshPrivateKey':
|
||||
ssh_private_key = decrypted_val
|
||||
if 'sshPrivateKey' in other_config and other_config['sshPrivateKey']:
|
||||
ssh_private_key = decrypt_sensitive_data(other_config['sshPrivateKey'])
|
||||
|
||||
# 检查是否为SSH URL
|
||||
is_ssh_url = GitSSHOperations.is_ssh_url(project.repository_url)
|
||||
|
|
@ -716,20 +709,7 @@ async def get_project_branches(
|
|||
gitea_token = settings.GITEA_TOKEN
|
||||
gitlab_token = settings.GITLAB_TOKEN
|
||||
|
||||
SENSITIVE_OTHER_FIELDS = ['githubToken', 'gitlabToken', 'giteaToken']
|
||||
|
||||
if config and config.other_config:
|
||||
import json
|
||||
other_config = json.loads(config.other_config)
|
||||
for field in SENSITIVE_OTHER_FIELDS:
|
||||
if field in other_config and other_config[field]:
|
||||
decrypted_val = decrypt_sensitive_data(other_config[field])
|
||||
if field == 'githubToken':
|
||||
github_token = decrypted_val
|
||||
elif field == 'gitlabToken':
|
||||
gitlab_token = decrypted_val
|
||||
elif field == 'giteaToken':
|
||||
gitea_token = decrypted_val
|
||||
# Git Token 始终来自 .env，不再从 config 中获取
|
||||
|
||||
repo_type = project.repository_type or "other"
|
||||
|
||||
|
|
@ -765,11 +745,13 @@ async def get_project_branches(
|
|||
return {"branches": branches, "default_branch": default_branch}
|
||||
|
||||
except Exception as e:
|
||||
import traceback
|
||||
error_msg = str(e)
|
||||
print(f"[Branch] 获取分支列表失败: {error_msg}")
|
||||
# 返回默认分支作为后备
|
||||
print(traceback.format_exc())
|
||||
# 返回默认分支作为后备，并包含错误详情
|
||||
return {
|
||||
"branches": [project.default_branch or "main"],
|
||||
"default_branch": project.default_branch or "main",
|
||||
"error": str(e)
|
||||
"error": error_msg
|
||||
}
|
||||
|
|
|
|||
|
|
@ -110,12 +110,20 @@ async def github_api(url: str, token: str = None) -> Any:
|
|||
headers["Authorization"] = f"Bearer {t}"
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
try:
|
||||
response = await client.get(url, headers=headers)
|
||||
if response.status_code == 403:
|
||||
raise Exception("GitHub API 403：请配置 GITHUB_TOKEN 或确认仓库权限/频率限制")
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"GitHub API {response.status_code}: {url}")
|
||||
return response.json()
|
||||
|
||||
data = response.json()
|
||||
if not isinstance(data, (list, dict)):
|
||||
print(f"[API] 警告: GitHub API 返回了非预期的格式: {type(data)}")
|
||||
return data
|
||||
except Exception as e:
|
||||
print(f"[API] GitHub API 调用失败: {url}, 错误: {e}")
|
||||
raise
|
||||
|
||||
|
||||
|
||||
|
|
@ -127,6 +135,7 @@ async def gitea_api(url: str, token: str = None) -> Any:
|
|||
headers["Authorization"] = f"token {t}"
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
try:
|
||||
response = await client.get(url, headers=headers)
|
||||
if response.status_code == 401:
|
||||
raise Exception("Gitea API 401：请配置 GITEA_TOKEN 或确认仓库权限")
|
||||
|
|
@ -134,7 +143,14 @@ async def gitea_api(url: str, token: str = None) -> Any:
|
|||
raise Exception("Gitea API 403：请确认仓库权限/频率限制")
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"Gitea API {response.status_code}: {url}")
|
||||
return response.json()
|
||||
|
||||
data = response.json()
|
||||
if not isinstance(data, (list, dict)):
|
||||
print(f"[API] 警告: Gitea API 返回了非预期的格式: {type(data)}")
|
||||
return data
|
||||
except Exception as e:
|
||||
print(f"[API] Gitea API 调用失败: {url}, 错误: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def gitlab_api(url: str, token: str = None) -> Any:
|
||||
|
|
@ -145,6 +161,7 @@ async def gitlab_api(url: str, token: str = None) -> Any:
|
|||
headers["PRIVATE-TOKEN"] = t
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
try:
|
||||
response = await client.get(url, headers=headers)
|
||||
if response.status_code == 401:
|
||||
raise Exception("GitLab API 401：请配置 GITLAB_TOKEN 或确认仓库权限")
|
||||
|
|
@ -152,7 +169,14 @@ async def gitlab_api(url: str, token: str = None) -> Any:
|
|||
raise Exception("GitLab API 403：请确认仓库权限/频率限制")
|
||||
if response.status_code != 200:
|
||||
raise Exception(f"GitLab API {response.status_code}: {url}")
|
||||
return response.json()
|
||||
|
||||
data = response.json()
|
||||
if not isinstance(data, (list, dict)):
|
||||
print(f"[API] 警告: GitLab API 返回了非预期的格式: {type(data)}")
|
||||
return data
|
||||
except Exception as e:
|
||||
print(f"[API] GitLab API 调用失败: {url}, 错误: {e}")
|
||||
raise
|
||||
|
||||
|
||||
async def fetch_file_content(url: str, headers: Dict[str, str] = None) -> Optional[str]:
|
||||
|
|
@ -175,7 +199,11 @@ async def get_github_branches(repo_url: str, token: str = None) -> List[str]:
|
|||
branches_url = f"https://api.github.com/repos/{owner}/{repo}/branches?per_page=100"
|
||||
branches_data = await github_api(branches_url, token)
|
||||
|
||||
return [b["name"] for b in branches_data]
|
||||
if not isinstance(branches_data, list):
|
||||
print(f"[Branch] 警告: 获取 GitHub 分支列表返回非列表数据: {branches_data}")
|
||||
return []
|
||||
|
||||
return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b]
|
||||
|
||||
|
||||
|
||||
|
|
@ -190,7 +218,11 @@ async def get_gitea_branches(repo_url: str, token: str = None) -> List[str]:
|
|||
branches_url = f"{base_url}/repos/{owner}/{repo}/branches"
|
||||
branches_data = await gitea_api(branches_url, token)
|
||||
|
||||
return [b["name"] for b in branches_data]
|
||||
if not isinstance(branches_data, list):
|
||||
print(f"[Branch] 警告: 获取 Gitea 分支列表返回非列表数据: {branches_data}")
|
||||
return []
|
||||
|
||||
return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b]
|
||||
|
||||
|
||||
async def get_gitlab_branches(repo_url: str, token: str = None) -> List[str]:
|
||||
|
|
@ -211,7 +243,11 @@ async def get_gitlab_branches(repo_url: str, token: str = None) -> List[str]:
|
|||
branches_url = f"{base_url}/projects/{project_path}/repository/branches?per_page=100"
|
||||
branches_data = await gitlab_api(branches_url, extracted_token)
|
||||
|
||||
return [b["name"] for b in branches_data]
|
||||
if not isinstance(branches_data, list):
|
||||
print(f"[Branch] 警告: 获取 GitLab 分支列表返回非列表数据: {branches_data}")
|
||||
return []
|
||||
|
||||
return [b["name"] for b in branches_data if isinstance(b, dict) and "name" in b]
|
||||
|
||||
|
||||
async def get_github_files(repo_url: str, branch: str, token: str = None, exclude_patterns: List[str] = None) -> List[Dict[str, str]]:
|
||||
|
|
@ -348,11 +384,10 @@ async def scan_repo_task(task_id: str, db_session_factory, user_config: dict = N
|
|||
print(f"📋 排除模式: {task_exclude_patterns}")
|
||||
|
||||
# 3. 获取文件列表
|
||||
# 从用户配置中读取 GitHub/GitLab Token（优先使用用户配置，然后使用系统配置）
|
||||
user_other_config = (user_config or {}).get('otherConfig', {})
|
||||
github_token = user_other_config.get('githubToken') or settings.GITHUB_TOKEN
|
||||
gitlab_token = user_other_config.get('gitlabToken') or settings.GITLAB_TOKEN
|
||||
gitea_token = user_other_config.get('giteaToken') or settings.GITEA_TOKEN
|
||||
# Git Token 始终来自系统默认（.env），逻辑锁定
|
||||
github_token = settings.GITHUB_TOKEN
|
||||
gitlab_token = settings.GITLAB_TOKEN
|
||||
gitea_token = settings.GITEA_TOKEN
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -114,10 +114,6 @@ export default function AgentModeSelector({
|
|||
className="sr-only"
|
||||
/>
|
||||
|
||||
{/* 推荐标签 */}
|
||||
<div className="absolute -top-2 -right-2 px-2 py-0.5 bg-violet-600 text-white text-xs font-bold uppercase font-mono rounded shadow-[0_0_10px_rgba(139,92,246,0.5)]">
|
||||
推荐
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<div className={cn(
|
||||
|
|
|
|||
|
|
@ -654,13 +654,17 @@ export function SystemConfig() {
|
|||
<TabsContent value="git" className="space-y-6">
|
||||
<div className="cyber-card p-6 space-y-6">
|
||||
<div className="space-y-2">
|
||||
<Label className="text-xs font-bold text-muted-foreground uppercase">GitHub Token (可选)</Label>
|
||||
<Label className="text-xs font-bold text-muted-foreground uppercase flex items-center justify-between">
|
||||
<span>GitHub Token (可选)</span>
|
||||
<span className="text-[10px] text-amber-500/80 normal-case font-normal border border-amber-500/30 px-1 rounded">.env 固定配置 (只读)</span>
|
||||
</Label>
|
||||
<Input
|
||||
type="password"
|
||||
type="text"
|
||||
value={config.githubToken}
|
||||
onChange={(e) => updateConfig('githubToken', e.target.value)}
|
||||
placeholder="ghp_xxxxxxxxxxxx"
|
||||
className="h-10 cyber-input"
|
||||
disabled
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
用于访问私有仓库。获取:{' '}
|
||||
|
|
@ -670,13 +674,17 @@ export function SystemConfig() {
|
|||
</p>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label className="text-xs font-bold text-muted-foreground uppercase">GitLab Token (可选)</Label>
|
||||
<Label className="text-xs font-bold text-muted-foreground uppercase flex items-center justify-between">
|
||||
<span>GitLab Token (可选)</span>
|
||||
<span className="text-[10px] text-amber-500/80 normal-case font-normal border border-amber-500/30 px-1 rounded">.env 固定配置 (只读)</span>
|
||||
</Label>
|
||||
<Input
|
||||
type="password"
|
||||
type="text"
|
||||
value={config.gitlabToken}
|
||||
onChange={(e) => updateConfig('gitlabToken', e.target.value)}
|
||||
placeholder="glpat-xxxxxxxxxxxx"
|
||||
className="h-10 cyber-input"
|
||||
disabled
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
用于访问私有仓库。获取:{' '}
|
||||
|
|
@ -686,13 +694,17 @@ export function SystemConfig() {
|
|||
</p>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label className="text-xs font-bold text-muted-foreground uppercase">Gitea Token (可选)</Label>
|
||||
<Label className="text-xs font-bold text-muted-foreground uppercase flex items-center justify-between">
|
||||
<span>Gitea Token (可选)</span>
|
||||
<span className="text-[10px] text-amber-500/80 normal-case font-normal border border-amber-500/30 px-1 rounded">.env 固定配置 (只读)</span>
|
||||
</Label>
|
||||
<Input
|
||||
type="password"
|
||||
type="text"
|
||||
value={config.giteaToken}
|
||||
onChange={(e) => updateConfig('giteaToken', e.target.value)}
|
||||
placeholder="sha1_xxxxxxxxxxxx"
|
||||
className="h-10 cyber-input"
|
||||
disabled
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
用于访问 Gitea 私有仓库。获取:{' '}
|
||||
|
|
@ -706,8 +718,9 @@ export function SystemConfig() {
|
|||
<Info className="w-4 h-4 text-sky-400" />
|
||||
提示
|
||||
</p>
|
||||
<p className="text-muted-foreground">• Git 分布式配置已锁定至后端 .env 文件</p>
|
||||
<p className="text-muted-foreground">• 如果需要修改，请编辑后端的 .env 环境文件并重启服务</p>
|
||||
<p className="text-muted-foreground">• 公开仓库无需配置 Token</p>
|
||||
<p className="text-muted-foreground">• 私有仓库需要配置对应平台的 Token</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
import asyncio
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add backend to sys.path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'backend')))
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
from app.services.scanner import get_gitea_branches
|
||||
|
||||
async def test_gitea_branches():
|
||||
print("Testing Gitea branch fetching with various mocked responses...")
|
||||
|
||||
repo_url = "http://gitea.example.com/owner/repo"
|
||||
token = "test-token"
|
||||
|
||||
test_cases = [
|
||||
{
|
||||
"name": "Normal list response",
|
||||
"mock_response": [{"name": "main"}, {"name": "develop"}],
|
||||
"expected_count": 2
|
||||
},
|
||||
{
|
||||
"name": "Null (None) response",
|
||||
"mock_response": None,
|
||||
"expected_count": 0
|
||||
},
|
||||
{
|
||||
"name": "Empty list response",
|
||||
"mock_response": [],
|
||||
"expected_count": 0
|
||||
},
|
||||
{
|
||||
"name": "Dict response (unexpected)",
|
||||
"mock_response": {"message": "Not found"},
|
||||
"expected_count": 0
|
||||
},
|
||||
{
|
||||
"name": "List with invalid items",
|
||||
"mock_response": [{"name": "main"}, "invalid", {"other": "data"}],
|
||||
"expected_count": 1
|
||||
}
|
||||
]
|
||||
|
||||
for case in test_cases:
|
||||
print(f"\nCase: {case['name']}")
|
||||
with patch('app.services.scanner.gitea_api', new_callable=AsyncMock) as mock_api:
|
||||
mock_api.return_value = case['mock_response']
|
||||
try:
|
||||
branches = await get_gitea_branches(repo_url, token)
|
||||
print(f"Result: {branches}")
|
||||
assert len(branches) == case['expected_count']
|
||||
print("✅ Pass")
|
||||
except Exception as e:
|
||||
print(f"❌ Fail with exception: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_gitea_branches())
|
||||
Loading…
Reference in New Issue