detailed logs.
Build and Push CodeReview / build (push) Waiting to run Details

This commit is contained in:
vinland100 2026-01-09 15:39:03 +08:00
parent dfefdfe432
commit f86e556d0d
1 changed files with 54 additions and 14 deletions

View File

@ -35,9 +35,8 @@ TEXT_EXTENSIONS = {
".py", ".js", ".ts", ".tsx", ".jsx", ".java", ".go", ".rs", ".py", ".js", ".ts", ".tsx", ".jsx", ".java", ".go", ".rs",
".cpp", ".c", ".h", ".cc", ".hh", ".hpp", ".hxx", ".cs", ".php", ".rb", ".cpp", ".c", ".h", ".cc", ".hh", ".hpp", ".hxx", ".cs", ".php", ".rb",
".kt", ".swift", ".dart", ".scala", ".sc", ".groovy", ".ktm", ".kts", ".kt", ".swift", ".dart", ".scala", ".sc", ".groovy", ".ktm", ".kts",
# .NET # .NET (逻辑代码)
".cshtml", ".vb", ".fs", ".fsi", ".fsx", ".sln", ".csproj", ".vbproj", ".cshtml", ".vb", ".fs", ".fsi", ".fsx", ".asax", ".master", ".ascx", ".asmx", ".svc",
".fsproj", ".config", ".asax", ".master", ".ascx", ".asmx", ".svc",
# 数据与配置 # 数据与配置
".json", ".yml", ".yaml", ".toml", ".xml", ".properties", ".conf", ".ini", ".json", ".yml", ".yaml", ".toml", ".xml", ".properties", ".conf", ".ini",
# 脚本与命令 # 脚本与命令
@ -47,7 +46,7 @@ TEXT_EXTENSIONS = {
".prisma", ".sol", ".dockerfile", ".tf", ".hcl", ".tfvars", ".prisma", ".sol", ".dockerfile", ".tf", ".hcl", ".tfvars",
# 其他 # 其他
".lua", ".hs", ".lhs", ".clj", ".cljs", ".cljc", ".edn", ".ex", ".exs", ".lua", ".hs", ".lhs", ".clj", ".cljs", ".cljc", ".edn", ".ex", ".exs",
".erl", ".hrl", ".m", ".mm", ".r", ".rmd", ".properties" ".erl", ".hrl", ".m", ".mm", ".r", ".rmd"
} }
# 排除的目录 # 排除的目录
@ -85,12 +84,18 @@ EXCLUDE_FILES = {
".DS_Store", "thumbs.db", "desktop.ini", "*.pem", "*.crt", "*.key", ".DS_Store", "thumbs.db", "desktop.ini", "*.pem", "*.crt", "*.key",
# 临时与日志 # 临时与日志
"*.log", "*.bak", "*.swp", "*.tmp", "tags", "*.log", "*.bak", "*.swp", "*.tmp", "tags",
# IDRE 与特定配置 # IDE 与项目配置 (非代码)
"*.suo", "*.user", "*.sln.docstates", "*.vshost.*", "*.pdb", "*.suo", "*.user", "*.sln", "*.csproj", "*.vbproj", "*.fsproj",
".ruby-version", ".nvmrc" "*.props", "*.targets", "*.resx", "*.sln.docstates", "*.vshost.*", "*.pdb",
"launchSettings.json", "dotnet-tools.json", ".ruby-version", ".nvmrc",
# 自动生成的代码 (噪声)
"*.Designer.cs", "*.Designer.vb", "*ModelSnapshot.cs", "*.generated.cs", "*.g.cs", "*.g.i.cs",
# 大型数据文件 (非代码)
"haarcascade_*.xml"
} }
class IndexUpdateMode(Enum): class IndexUpdateMode(Enum):
"""索引更新模式""" """索引更新模式"""
FULL = "full" # 全量重建:删除旧索引,完全重新索引 FULL = "full" # 全量重建:删除旧索引,完全重新索引
@ -994,8 +999,15 @@ class CodeIndexer:
# 🔥 详细打印所有待索引的文件名,方便调试 (满足用户需求) # 🔥 详细打印所有待索引的文件名,方便调试 (满足用户需求)
if files: if files:
relative_files = [os.path.relpath(f, directory) for f in files] file_infos = []
logger.info(f"📄 待索引文件列表: {', '.join(sorted(relative_files))}") # 按相对路径排序
sorted_files = sorted(files, key=lambda f: os.path.relpath(f, directory))
for f in sorted_files:
rel = os.path.relpath(f, directory)
size_str = self._get_file_size_str(f)
file_infos.append(f"{rel} ({size_str})")
logger.info(f"📄 待索引文件列表: {', '.join(file_infos)}")
yield progress yield progress
@ -1009,6 +1021,8 @@ class CodeIndexer:
progress.current_file = relative_path progress.current_file = relative_path
# 异步读取文件 # 异步读取文件
file_size = os.path.getsize(file_path)
logger.info(f"📄 正在处理: {relative_path} (大小: {file_size / 1024:.2f} KB)")
content = await asyncio.to_thread(self._read_file_sync, file_path) content = await asyncio.to_thread(self._read_file_sync, file_path)
if not content.strip(): if not content.strip():
progress.processed_files += 1 progress.processed_files += 1
@ -1140,9 +1154,11 @@ class CodeIndexer:
# 🔥 详细打印新增、更新和删除的文件名,方便调试 (满足用户需求) # 🔥 详细打印新增、更新和删除的文件名,方便调试 (满足用户需求)
if files_to_add: if files_to_add:
logger.info(f"🆕 新增文件 ({len(files_to_add)}): {', '.join(sorted(list(files_to_add)))}") file_infos = [f"{rel} ({self._get_file_size_str(current_file_map.get(rel))})" for rel in sorted(list(files_to_add))]
logger.info(f"🆕 新增文件 ({len(files_to_add)}): {', '.join(file_infos)}")
if files_to_update: if files_to_update:
logger.info(f"🔄 更新文件 ({len(files_to_update)}): {', '.join(sorted(list(files_to_update)))}") file_infos = [f"{rel} ({self._get_file_size_str(current_file_map.get(rel))})" for rel in sorted(list(files_to_update))]
logger.info(f"🔄 更新文件 ({len(files_to_update)}): {', '.join(file_infos)}")
if files_to_delete: if files_to_delete:
logger.info(f"🗑️ 删除文件 ({len(files_to_delete)}): {', '.join(sorted(list(files_to_delete)))}") logger.info(f"🗑️ 删除文件 ({len(files_to_delete)}): {', '.join(sorted(list(files_to_delete)))}")
@ -1171,6 +1187,8 @@ class CodeIndexer:
try: try:
# 异步读取文件 # 异步读取文件
file_size = os.path.getsize(file_path)
logger.info(f"📄 正在处理: {relative_path} (大小: {file_size / 1024:.2f} KB)")
content = await asyncio.to_thread(self._read_file_sync, file_path) content = await asyncio.to_thread(self._read_file_sync, file_path)
if not content.strip(): if not content.strip():
@ -1305,6 +1323,8 @@ class CodeIndexer:
progress.current_file = file_path progress.current_file = file_path
try: try:
file_size = len(content.encode('utf-8'))
logger.info(f"📄 正在处理: {file_path} (大小: {file_size / 1024:.2f} KB)")
if not content.strip(): if not content.strip():
progress.processed_files += 1 progress.processed_files += 1
progress.skipped_files += 1 progress.skipped_files += 1
@ -1381,10 +1401,14 @@ class CodeIndexer:
chunks = unique_chunks chunks = unique_chunks
# 获取涉及的文件名用于日志展示
file_paths = sorted(list(set(chunk.file_path for chunk in chunks)))
file_label = f"[{file_paths[0]}]" if len(file_paths) == 1 else f"[{len(file_paths)} 个文件]"
# 准备嵌入文本 # 准备嵌入文本
texts = [chunk.to_embedding_text() for chunk in chunks] texts = [chunk.to_embedding_text() for chunk in chunks]
logger.info(f"🔢 生成 {len(texts)} 个代码块的嵌入向量...") logger.info(f"🔢 文件: {file_label} 生成 {len(texts)} 个代码块的嵌入向量...")
# 批量嵌入(带进度回调和取消检查) # 批量嵌入(带进度回调和取消检查)
embeddings = await self.embedding_service.embed_batch( embeddings = await self.embedding_service.embed_batch(
@ -1400,7 +1424,7 @@ class CodeIndexer:
metadatas = [chunk.to_dict() for chunk in chunks] metadatas = [chunk.to_dict() for chunk in chunks]
# 添加到向量存储 # 添加到向量存储
logger.info(f"💾 添加 {len(chunks)} 个代码块到向量存储...") logger.info(f"💾 文件: {file_label} 添加 {len(chunks)} 个代码块到向量存储...")
if use_upsert: if use_upsert:
await self.vector_store.upsert_documents( await self.vector_store.upsert_documents(
@ -1417,7 +1441,23 @@ class CodeIndexer:
metadatas=metadatas, metadatas=metadatas,
) )
logger.info(f"✅ 索引 {len(chunks)} 个代码块成功") logger.info(f"✅ 文件:{file_label} 索引 {len(chunks)} 个代码块成功")
@staticmethod
def _get_file_size_str(file_path: Optional[str]) -> str:
"""获取文件大小的可读字符串"""
if not file_path or not os.path.exists(file_path):
return "unknown"
try:
size = os.path.getsize(file_path)
if size < 1024:
return f"{size} B"
elif size < 1024 * 1024:
return f"{size / 1024:.2f} KB"
else:
return f"{size / (1024 * 1024):.2f} MB"
except Exception:
return "error"
def _collect_files( def _collect_files(
self, self,