text_search.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792
  1. from fastapi import APIRouter, HTTPException, Depends
  2. from pydantic import BaseModel, Field, validator
  3. from typing import List, Optional
  4. from service.trunks_service import TrunksService
  5. from utils.sentence_util import SentenceUtil
  6. from utils.vector_distance import VectorDistance
  7. from model.response import StandardResponse
  8. from utils.vectorizer import Vectorizer
  9. # from utils.find_text_in_pdf import find_text_in_pdf
  10. import os
  11. DISTANCE_THRESHOLD = 0.73
  12. import logging
  13. import time
  14. from db.session import get_db
  15. from sqlalchemy.orm import Session
  16. from service.kg_node_service import KGNodeService
  17. from service.kg_prop_service import KGPropService
  18. from service.kg_edge_service import KGEdgeService
  19. from cachetools import TTLCache
  20. # 使用TextSimilarityFinder进行文本相似度匹配
  21. from utils.text_similarity import TextSimilarityFinder
  22. logger = logging.getLogger(__name__)
  23. router = APIRouter(tags=["Text Search"])
  24. # 创建全局缓存实例
  25. cache = TTLCache(maxsize=1000, ttl=3600)
  26. class TextSearchRequest(BaseModel):
  27. text: str
  28. conversation_id: Optional[str] = None
  29. need_convert: Optional[bool] = False
  30. class TextCompareRequest(BaseModel):
  31. sentence: str
  32. text: str
  33. class TextMatchRequest(BaseModel):
  34. text: str = Field(..., min_length=1, max_length=10000, description="需要搜索的文本内容")
  35. @validator('text')
  36. def validate_text(cls, v):
  37. # 保留所有可打印字符、换行符和中文字符
  38. v = ''.join(char for char in v if char.isprintable() or char in '\n\r')
  39. # 转义JSON特殊字符
  40. # 先处理反斜杠,避免后续转义时出现问题
  41. v = v.replace('\\', '\\\\')
  42. # 处理引号和其他特殊字符
  43. v = v.replace('"', '\\"')
  44. v = v.replace('/', '\\/')
  45. # 处理控制字符
  46. v = v.replace('\n', '\\n')
  47. v = v.replace('\r', '\\r')
  48. v = v.replace('\t', '\\t')
  49. v = v.replace('\b', '\\b')
  50. v = v.replace('\f', '\\f')
  51. # 处理Unicode转义
  52. # v = v.replace('\u', '\\u')
  53. return v
  54. class TextCompareMultiRequest(BaseModel):
  55. origin: str
  56. similar: str
  57. class NodePropsSearchRequest(BaseModel):
  58. node_id: int
  59. props_ids: List[int]
  60. symptoms: Optional[List[str]] = None
  61. @router.post("/kgrt_api/text/clear_cache", response_model=StandardResponse)
  62. async def clear_cache():
  63. try:
  64. # 清除全局缓存
  65. cache.clear()
  66. return StandardResponse(success=True, data={"message": "缓存已清除"})
  67. except Exception as e:
  68. logger.error(f"清除缓存失败: {str(e)}")
  69. raise HTTPException(status_code=500, detail=str(e))
  70. @router.post("/kgrt_api/text/search", response_model=StandardResponse)
  71. @router.post("/knowledge/text/search", response_model=StandardResponse)
  72. async def search_text(request: TextSearchRequest):
  73. try:
  74. #判断request.text是否为json格式,如果是,使用JsonToText的convert方法转换为text
  75. if request.text.startswith('{') and request.text.endswith('}'):
  76. from utils.json_to_text import JsonToTextConverter
  77. converter = JsonToTextConverter()
  78. request.text = converter.convert(request.text)
  79. # 使用TextSplitter拆分文本
  80. sentences = SentenceUtil.split_text(request.text)
  81. if not sentences:
  82. return StandardResponse(success=True, data={"answer": "", "references": []})
  83. # 初始化服务和结果列表
  84. trunks_service = TrunksService()
  85. result_sentences = []
  86. all_references = []
  87. reference_index = 1
  88. # 根据conversation_id获取缓存结果
  89. cached_results = trunks_service.get_cached_result(request.conversation_id) if request.conversation_id else []
  90. for sentence in sentences:
  91. # if request.need_convert:
  92. sentence = sentence.replace("\n", "<br>")
  93. if len(sentence) < 10:
  94. result_sentences.append(sentence)
  95. continue
  96. if cached_results:
  97. # 如果有缓存结果,计算向量距离
  98. min_distance = float('inf')
  99. best_result = None
  100. sentence_vector = Vectorizer.get_embedding(sentence)
  101. for cached_result in cached_results:
  102. content_vector = cached_result['embedding']
  103. distance = VectorDistance.calculate_distance(sentence_vector, content_vector)
  104. if distance < min_distance:
  105. min_distance = distance
  106. best_result = {**cached_result, 'distance': distance}
  107. if best_result and best_result['distance'] < DISTANCE_THRESHOLD:
  108. search_results = [best_result]
  109. else:
  110. search_results = []
  111. else:
  112. # 如果没有缓存结果,进行向量搜索
  113. search_results = trunks_service.search_by_vector(
  114. text=sentence,
  115. limit=1,
  116. type='trunk'
  117. )
  118. # 处理搜索结果
  119. for search_result in search_results:
  120. distance = search_result.get("distance", DISTANCE_THRESHOLD)
  121. if distance >= DISTANCE_THRESHOLD:
  122. result_sentences.append(sentence)
  123. continue
  124. # 检查是否已存在相同引用
  125. existing_ref = next((ref for ref in all_references if ref["id"] == search_result["id"]), None)
  126. current_index = reference_index
  127. if existing_ref:
  128. current_index = int(existing_ref["index"])
  129. else:
  130. # 添加到引用列表
  131. # 从referrence中提取文件名
  132. file_name = ""
  133. referrence = search_result.get("referrence", "")
  134. if referrence and "/books/" in referrence:
  135. file_name = referrence.split("/books/")[-1]
  136. # 去除文件扩展名
  137. file_name = os.path.splitext(file_name)[0]
  138. reference = {
  139. "index": str(reference_index),
  140. "id": search_result["id"],
  141. "content": search_result["content"],
  142. "file_path": search_result.get("file_path", ""),
  143. "title": search_result.get("title", ""),
  144. "distance": distance,
  145. "file_name": file_name,
  146. "referrence": referrence
  147. }
  148. all_references.append(reference)
  149. reference_index += 1
  150. # 添加引用标记
  151. if sentence.endswith('<br>'):
  152. # 如果有多个<br>,在所有<br>前添加^[current_index]^
  153. result_sentence = sentence.replace('<br>', f'^[{current_index}]^<br>')
  154. else:
  155. # 直接在句子末尾添加^[current_index]^
  156. result_sentence = f'{sentence}^[{current_index}]^'
  157. result_sentences.append(result_sentence)
  158. # 组装返回数据
  159. response_data = {
  160. "answer": result_sentences,
  161. "references": all_references
  162. }
  163. return StandardResponse(success=True, data=response_data)
  164. except Exception as e:
  165. logger.error(f"Text search failed: {str(e)}")
  166. raise HTTPException(status_code=500, detail=str(e))
  167. @router.post("/kgrt_api/text/match", response_model=StandardResponse)
  168. @router.post("/knowledge/text/match", response_model=StandardResponse)
  169. async def match_text(request: TextCompareRequest):
  170. try:
  171. sentences = SentenceUtil.split_text(request.text)
  172. sentence_vector = Vectorizer.get_embedding(request.sentence)
  173. min_distance = float('inf')
  174. best_sentence = ""
  175. result_sentences = []
  176. for temp in sentences:
  177. result_sentences.append(temp)
  178. if len(temp) < 10:
  179. continue
  180. temp_vector = Vectorizer.get_embedding(temp)
  181. distance = VectorDistance.calculate_distance(sentence_vector, temp_vector)
  182. if distance < min_distance and distance < DISTANCE_THRESHOLD:
  183. min_distance = distance
  184. best_sentence = temp
  185. for i in range(len(result_sentences)):
  186. result_sentences[i] = {"sentence": result_sentences[i], "matched": False}
  187. if result_sentences[i]["sentence"] == best_sentence:
  188. result_sentences[i]["matched"] = True
  189. return StandardResponse(success=True, records=result_sentences)
  190. except Exception as e:
  191. logger.error(f"Text comparison failed: {str(e)}")
  192. raise HTTPException(status_code=500, detail=str(e))
  193. @router.post("/kgrt_api/text/mr_search", response_model=StandardResponse)
  194. @router.post("/knowledge/text/mr_search", response_model=StandardResponse)
  195. async def mr_search_text_content(request: TextMatchRequest):
  196. try:
  197. # 初始化服务
  198. trunks_service = TrunksService()
  199. # 获取文本向量并搜索相似内容
  200. search_results = trunks_service.search_by_vector(
  201. text=request.text,
  202. limit=10,
  203. type="mr"
  204. )
  205. # 处理搜索结果
  206. records = []
  207. for result in search_results:
  208. distance = result.get("distance", DISTANCE_THRESHOLD)
  209. if distance >= DISTANCE_THRESHOLD:
  210. continue
  211. # 添加到引用列表
  212. record = {
  213. "content": result["content"],
  214. "file_path": result.get("file_path", ""),
  215. "title": result.get("title", ""),
  216. "distance": distance,
  217. }
  218. records.append(record)
  219. # 组装返回数据
  220. response_data = {
  221. "records": records
  222. }
  223. return StandardResponse(success=True, data=response_data)
  224. except Exception as e:
  225. logger.error(f"Mr search failed: {str(e)}")
  226. raise HTTPException(status_code=500, detail=str(e))
  227. @router.post("/kgrt_api/text/mr_match", response_model=StandardResponse)
  228. @router.post("/knowledge/text/mr_match", response_model=StandardResponse)
  229. async def compare_text(request: TextCompareMultiRequest):
  230. start_time = time.time()
  231. try:
  232. # 拆分两段文本
  233. origin_sentences = SentenceUtil.split_text(request.origin)
  234. similar_sentences = SentenceUtil.split_text(request.similar)
  235. end_time = time.time()
  236. logger.info(f"mr_match接口处理文本耗时: {(end_time - start_time) * 1000:.2f}ms")
  237. # 初始化结果列表
  238. origin_results = []
  239. # 过滤短句并预计算向量
  240. valid_origin_sentences = [(sent, len(sent) >= 10) for sent in origin_sentences]
  241. valid_similar_sentences = [(sent, len(sent) >= 10) for sent in similar_sentences]
  242. # 初始化similar_results,所有matched设为False
  243. similar_results = [{"sentence": sent, "matched": False} for sent, _ in valid_similar_sentences]
  244. # 批量获取向量
  245. origin_vectors = {}
  246. similar_vectors = {}
  247. origin_batch = [sent for sent, is_valid in valid_origin_sentences if is_valid]
  248. similar_batch = [sent for sent, is_valid in valid_similar_sentences if is_valid]
  249. if origin_batch:
  250. origin_embeddings = [Vectorizer.get_embedding(sent) for sent in origin_batch]
  251. origin_vectors = dict(zip(origin_batch, origin_embeddings))
  252. if similar_batch:
  253. similar_embeddings = [Vectorizer.get_embedding(sent) for sent in similar_batch]
  254. similar_vectors = dict(zip(similar_batch, similar_embeddings))
  255. end_time = time.time()
  256. logger.info(f"mr_match接口处理向量耗时: {(end_time - start_time) * 1000:.2f}ms")
  257. # 处理origin文本
  258. for origin_sent, is_valid in valid_origin_sentences:
  259. if not is_valid:
  260. origin_results.append({"sentence": origin_sent, "matched": False})
  261. continue
  262. origin_vector = origin_vectors[origin_sent]
  263. matched = False
  264. # 优化的相似度计算
  265. for i, similar_result in enumerate(similar_results):
  266. if similar_result["matched"]:
  267. continue
  268. similar_sent = similar_result["sentence"]
  269. if len(similar_sent) < 10:
  270. continue
  271. similar_vector = similar_vectors.get(similar_sent)
  272. if not similar_vector:
  273. continue
  274. distance = VectorDistance.calculate_distance(origin_vector, similar_vector)
  275. if distance < DISTANCE_THRESHOLD:
  276. matched = True
  277. similar_results[i]["matched"] = True
  278. break
  279. origin_results.append({"sentence": origin_sent, "matched": matched})
  280. response_data = {
  281. "origin": origin_results,
  282. "similar": similar_results
  283. }
  284. end_time = time.time()
  285. logger.info(f"mr_match接口耗时: {(end_time - start_time) * 1000:.2f}ms")
  286. return StandardResponse(success=True, data=response_data)
  287. except Exception as e:
  288. end_time = time.time()
  289. logger.error(f"Text comparison failed: {str(e)}")
  290. logger.info(f"mr_match接口耗时: {(end_time - start_time) * 1000:.2f}ms")
  291. raise HTTPException(status_code=500, detail=str(e))
  292. def _check_cache(node_id: int) -> Optional[dict]:
  293. """检查并返回缓存结果"""
  294. cache_key = f"xunzheng_{node_id}"
  295. cached_result = cache.get(cache_key)
  296. if cached_result:
  297. logger.info(f"从缓存获取结果,node_id: {node_id}")
  298. return cached_result
  299. return None
  300. def _get_node_info(node_service: KGNodeService, node_id: int) -> dict:
  301. """获取并验证节点信息"""
  302. node = node_service.get_node(node_id)
  303. if not node:
  304. raise ValueError(f"节点不存在: {node_id}")
  305. return {
  306. "id": node_id,
  307. "name": node.get('name', ''),
  308. "category": node.get('category', ''),
  309. "props": [],
  310. "files": [],
  311. "distance": 0
  312. }
  313. def _process_search_result(search_result: dict, reference_index: int) -> tuple[dict, str]:
  314. """处理搜索结果,返回引用信息和文件名"""
  315. file_name = ""
  316. referrence = search_result.get("referrence", "")
  317. if referrence and "/books/" in referrence:
  318. file_name = referrence.split("/books/")[-1]
  319. file_name = os.path.splitext(file_name)[0]
  320. reference = {
  321. "index": str(reference_index),
  322. "id": search_result["id"],
  323. "content": search_result["content"],
  324. "file_path": search_result.get("file_path", ""),
  325. "title": search_result.get("title", ""),
  326. "distance": search_result.get("distance", DISTANCE_THRESHOLD),
  327. "page_no": search_result.get("page_no", ""),
  328. "file_name": file_name,
  329. "referrence": referrence
  330. }
  331. return reference, file_name
  332. def _get_file_type(file_name: str) -> str:
  333. """根据文件名确定文件类型"""
  334. file_name_lower = file_name.lower()
  335. if file_name_lower.endswith(".pdf"):
  336. return "pdf"
  337. elif file_name_lower.endswith((".doc", ".docx")):
  338. return "doc"
  339. elif file_name_lower.endswith((".xls", ".xlsx")):
  340. return "excel"
  341. elif file_name_lower.endswith((".ppt", ".pptx")):
  342. return "ppt"
  343. return "other"
  344. def _process_sentence_search(node_name: str, prop_title: str, sentences: list, trunks_service: TrunksService) -> tuple[list, list]:
  345. """处理句子搜索,返回结果句子和引用列表"""
  346. result_sentences = []
  347. all_references = []
  348. reference_index = 1
  349. i = 0
  350. while i < len(sentences):
  351. sentence = sentences[i]
  352. search_text = f"{node_name}:{prop_title}:{sentence}"
  353. # if len(sentence) < 10 and i + 1 < len(sentences):
  354. # next_sentence = sentences[i + 1]
  355. # # result_sentences.append({"sentence": sentence, "flag": ""})
  356. # search_text = f"{node_name}:{prop_title}:{sentence} {next_sentence}"
  357. # i += 1
  358. # elif len(sentence) < 10:
  359. # result_sentences.append({"sentence": sentence, "flag": ""})
  360. # i += 1
  361. # continue
  362. # else:
  363. i += 1
  364. # 使用向量搜索获取相似内容
  365. search_results = trunks_service.search_by_vector(
  366. text=search_text,
  367. limit=500,
  368. type='trunk',
  369. distance=0.7
  370. )
  371. # 准备语料库数据
  372. trunk_texts = []
  373. trunk_ids = []
  374. # 创建一个字典来存储trunk的详细信息
  375. trunk_details = {}
  376. for trunk in search_results:
  377. trunk_texts.append(trunk.get('content'))
  378. trunk_ids.append(trunk.get('id'))
  379. # 缓存trunk的详细信息
  380. trunk_details[trunk.get('id')] = {
  381. 'id': trunk.get('id'),
  382. 'content': trunk.get('content'),
  383. 'file_path': trunk.get('file_path'),
  384. 'title': trunk.get('title'),
  385. 'referrence': trunk.get('referrence'),
  386. 'page_no': trunk.get('page_no')
  387. }
  388. if len(trunk_texts) == 0:
  389. continue
  390. # 初始化TextSimilarityFinder并加载语料库
  391. similarity_finder = TextSimilarityFinder(method='tfidf', use_jieba=True)
  392. similarity_finder.load_corpus(trunk_texts, trunk_ids)
  393. # 使用TextSimilarityFinder进行相似度匹配
  394. similar_results = similarity_finder.find_most_similar(search_text, top_n=1)
  395. if not similar_results: # 设置相似度阈值
  396. result_sentences.append({"sentence": sentence, "flag": ""})
  397. continue
  398. # 获取最相似的文本对应的trunk_id
  399. trunk_id = similar_results[0]['path']
  400. # 从缓存中获取trunk详细信息
  401. trunk_info = trunk_details.get(trunk_id)
  402. if trunk_info:
  403. search_result = {
  404. **trunk_info,
  405. 'distance': similar_results[0]['similarity'] # 转换相似度为距离
  406. }
  407. # 检查相似度是否达到阈值
  408. if search_result['distance'] >= DISTANCE_THRESHOLD:
  409. result_sentences.append({"sentence": sentence, "flag": ""})
  410. continue
  411. # 检查是否已存在相同引用
  412. existing_ref = next((ref for ref in all_references if ref["id"] == search_result["id"]), None)
  413. current_index = int(existing_ref["index"]) if existing_ref else reference_index
  414. if not existing_ref:
  415. reference, _ = _process_search_result(search_result, reference_index)
  416. all_references.append(reference)
  417. reference_index += 1
  418. result_sentences.append({"sentence": sentence, "flag": str(current_index)})
  419. return result_sentences, all_references
  420. def _mark_symptoms(text: str, symptom_list: List[str]) -> str:
  421. """处理症状标记"""
  422. if not symptom_list:
  423. return text
  424. marked_sentence = text
  425. # 创建一个标记位置的列表,记录每个位置是否已被标记
  426. marked_positions = [False] * len(marked_sentence)
  427. # 创建一个列表来存储已处理的症状
  428. processed_symptoms = []
  429. for symptom in symptom_list:
  430. # 检查是否已处理过该症状或其子集
  431. if any(symptom in processed_sym or processed_sym in symptom for processed_sym in processed_symptoms):
  432. continue
  433. # 查找所有匹配位置
  434. start_pos = 0
  435. while True:
  436. pos = marked_sentence.find(symptom, start_pos)
  437. if pos == -1:
  438. break
  439. # 检查这个位置是否已被标记
  440. if not any(marked_positions[pos:pos + len(symptom)]):
  441. # 标记这个范围的所有位置
  442. for i in range(pos, pos + len(symptom)):
  443. marked_positions[i] = True
  444. # 替换文本
  445. marked_sentence = marked_sentence[:pos] + f'<i style="color:red;">{symptom}</i>' + marked_sentence[pos + len(symptom):]
  446. # 将成功标记的症状添加到已处理列表中
  447. if symptom not in processed_symptoms:
  448. processed_symptoms.append(symptom)
  449. # 更新标记位置数组以适应新插入的标签
  450. new_positions = [False] * (len('<i style="color:red;">') + len('</i>'))
  451. marked_positions = marked_positions[:pos] + new_positions + marked_positions[pos:]
  452. start_pos = pos + len('<i style="color:red;">') + len(symptom) + len('</i>')
  453. return marked_sentence
  454. @router.post("/kgrt_api/text/eb_search", response_model=StandardResponse)
  455. @router.post("/knowledge/text/eb_search", response_model=StandardResponse)
  456. async def node_props_search(request: NodePropsSearchRequest, db: Session = Depends(get_db)):
  457. try:
  458. start_time = time.time()
  459. # 检查缓存
  460. cached_result = _check_cache(request.node_id)
  461. if cached_result:
  462. # 如果有症状列表,处理症状标记
  463. if request.symptoms:
  464. symptom_list = []
  465. try:
  466. # 初始化服务
  467. node_service = KGNodeService(db)
  468. edge_service = KGEdgeService(db)
  469. for symptom in request.symptoms:
  470. # 添加原始症状
  471. symptom_list.append(symptom)
  472. try:
  473. # 获取症状节点
  474. symptom_node = node_service.get_node_by_name_category(symptom, '症状')
  475. # 获取症状相关同义词(包括1.0和2.0版本)
  476. for category in ['症状同义词', '症状同义词2.0']:
  477. edges = edge_service.get_edges_by_nodes(src_id=symptom_node['id'], category=category)
  478. if edges:
  479. # 添加同义词
  480. for edge in edges:
  481. if edge['dest_node'] and edge['dest_node'].get('name'):
  482. symptom_list.append(edge['dest_node']['name'])
  483. except ValueError:
  484. # 如果找不到节点,只添加原始症状
  485. continue
  486. # 按照字符长度进行倒序排序
  487. symptom_list.sort(key=len, reverse=True)
  488. # 处理缓存结果中的症状标记
  489. for prop in cached_result.get('props', []):
  490. if prop.get('prop_title') == '临床表现' and 'answer' in prop:
  491. for answer in prop['answer']:
  492. answer['sentence'] = _mark_symptoms(answer['sentence'], symptom_list)
  493. except Exception as e:
  494. logger.error(f"处理症状标记失败: {str(e)}")
  495. return StandardResponse(success=True, data=cached_result)
  496. # 初始化服务
  497. trunks_service = TrunksService()
  498. node_service = KGNodeService(db)
  499. prop_service = KGPropService(db)
  500. edge_service = KGEdgeService(db)
  501. # 获取节点信息
  502. result = _get_node_info(node_service, request.node_id)
  503. node_name = result["name"]
  504. # 处理症状列表
  505. symptom_list = []
  506. if request.symptoms:
  507. for symptom in request.symptoms:
  508. try:
  509. # 添加原始症状
  510. symptom_list.append(symptom)
  511. # 获取症状节点
  512. symptom_node = node_service.get_node_by_name_category(symptom, '症状')
  513. # 获取症状相关同义词(包括1.0和2.0版本)
  514. for category in ['症状同义词', '症状同义词2.0']:
  515. edges = edge_service.get_edges_by_nodes(src_id=symptom_node['id'], category=category)
  516. if edges:
  517. # 添加同义词
  518. for edge in edges:
  519. if edge['dest_node'] and edge['dest_node'].get('name'):
  520. symptom_list.append(edge['dest_node']['name'])
  521. except ValueError:
  522. # 如果找不到节点,只添加原始症状
  523. continue
  524. # 按照字符长度进行倒序排序
  525. symptom_list.sort(key=len, reverse=True)
  526. # 遍历props_ids查询属性信息
  527. for prop_id in request.props_ids:
  528. prop = prop_service.get_prop_by_id(prop_id)
  529. if not prop:
  530. logger.warning(f"属性不存在: {prop_id}")
  531. continue
  532. prop_title = prop.get('prop_title', '')
  533. prop_value = prop.get('prop_value', '')
  534. # 创建属性结果对象
  535. prop_result = {
  536. "id": prop_id,
  537. "category": prop.get('category', 0),
  538. "prop_name": prop.get('prop_name', ''),
  539. "prop_value": prop_value,
  540. "prop_title": prop_title,
  541. "type": prop.get('type', 1)
  542. }
  543. result["props"].append(prop_result)
  544. # 如果prop_value为'无',则跳过搜索
  545. if prop_value == '无':
  546. prop_result["answer"] = [{
  547. "sentence": prop_value,
  548. "flag": ""
  549. }]
  550. continue
  551. # 先用完整的prop_value进行搜索
  552. search_text = f"{node_name}:{prop_title}:{prop_value}"
  553. # 使用向量搜索获取相似内容
  554. search_results = trunks_service.search_by_vector(
  555. text=search_text,
  556. limit=500,
  557. type='trunk',
  558. distance=0.7
  559. )
  560. # 准备语料库数据
  561. trunk_texts = []
  562. trunk_ids = []
  563. # 创建一个字典来存储trunk的详细信息
  564. trunk_details = {}
  565. for trunk in search_results:
  566. trunk_texts.append(trunk.get('content'))
  567. trunk_ids.append(trunk.get('id'))
  568. # 缓存trunk的详细信息
  569. trunk_details[trunk.get('id')] = {
  570. 'id': trunk.get('id'),
  571. 'content': trunk.get('content'),
  572. 'file_path': trunk.get('file_path'),
  573. 'title': trunk.get('title'),
  574. 'referrence': trunk.get('referrence'),
  575. 'page_no': trunk.get('page_no')
  576. }
  577. if len(trunk_texts)==0:
  578. continue
  579. # 初始化TextSimilarityFinder并加载语料库
  580. similarity_finder = TextSimilarityFinder(method='tfidf', use_jieba=True)
  581. similarity_finder.load_corpus(trunk_texts, trunk_ids)
  582. similar_results = similarity_finder.find_most_similar(search_text, top_n=1)
  583. # 处理搜索结果
  584. if similar_results and similar_results[0]['similarity']>=0.3: # 设置相似度阈值
  585. # 获取最相似的文本对应的trunk_id
  586. trunk_id = similar_results[0]['path']
  587. # 从缓存中获取trunk详细信息
  588. trunk_info = trunk_details.get(trunk_id)
  589. if trunk_info:
  590. search_result = {
  591. **trunk_info,
  592. 'distance': similar_results[0]['similarity'] # 转换相似度为距离
  593. }
  594. reference, _ = _process_search_result(search_result, 1)
  595. prop_result["references"] = [reference]
  596. prop_result["answer"] = [{
  597. "sentence": prop_value,
  598. "flag": "1"
  599. }]
  600. else:
  601. # 如果整体搜索没有找到匹配结果,则进行句子拆分搜索
  602. sentences = SentenceUtil.split_text(prop_value,10)
  603. else:
  604. # 如果整体搜索没有找到匹配结果,则进行句子拆分搜索
  605. sentences = SentenceUtil.split_text(prop_value,10)
  606. result_sentences, references = _process_sentence_search(
  607. node_name, prop_title, sentences, trunks_service
  608. )
  609. if references:
  610. prop_result["references"] = references
  611. if result_sentences:
  612. prop_result["answer"] = result_sentences
  613. # 处理文件信息
  614. all_files = set()
  615. file_index_map = {}
  616. file_index = 1
  617. # 收集文件信息
  618. for prop_result in result["props"]:
  619. if "references" not in prop_result:
  620. continue
  621. for ref in prop_result["references"]:
  622. referrence = ref.get("referrence", "")
  623. if not (referrence and "/books/" in referrence):
  624. continue
  625. file_name = referrence.split("/books/")[-1]
  626. if not file_name:
  627. continue
  628. file_type = _get_file_type(file_name)
  629. if file_name not in file_index_map:
  630. file_index_map[file_name] = file_index
  631. file_index += 1
  632. all_files.add((file_name, file_type))
  633. # 更新引用索引
  634. for prop_result in result["props"]:
  635. if "references" not in prop_result:
  636. continue
  637. for ref in prop_result["references"]:
  638. referrence = ref.get("referrence", "")
  639. if referrence and "/books/" in referrence:
  640. file_name = referrence.split("/books/")[-1]
  641. if file_name in file_index_map:
  642. ref["index"] = f"{file_index_map[file_name]}-{ref['index']}"
  643. # 更新answer中的flag
  644. if "answer" in prop_result:
  645. for sentence in prop_result["answer"]:
  646. if sentence["flag"]:
  647. for ref in prop_result["references"]:
  648. if ref["index"].endswith(f"-{sentence['flag']}"):
  649. sentence["flag"] = ref["index"]
  650. break
  651. # 添加文件信息到结果
  652. result["files"] = sorted([{
  653. "file_name": file_name,
  654. "file_type": file_type,
  655. "index": str(file_index_map[file_name])
  656. } for file_name, file_type in all_files], key=lambda x: int(x["index"]))
  657. # 缓存结果
  658. cache_key = f"xunzheng_{request.node_id}"
  659. cache[cache_key] = result
  660. # 处理症状标记
  661. if request.symptoms:
  662. for prop in result.get('props', []):
  663. if prop.get('prop_title') == '临床表现' and 'answer' in prop:
  664. for answer in prop['answer']:
  665. answer['sentence'] = _mark_symptoms(answer['sentence'], symptom_list)
  666. end_time = time.time()
  667. logger.info(f"node_props_search接口耗时: {(end_time - start_time) * 1000:.2f}ms")
  668. return StandardResponse(success=True, data=result)
  669. except Exception as e:
  670. logger.error(f"Node props search failed: {str(e)}")
  671. raise HTTPException(status_code=500, detail=str(e))
  672. text_search_router = router