kg_node_service.py 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222
  1. from sqlalchemy.orm import Session
  2. from typing import Optional
  3. from model.kg_node import KGNode
  4. from db.session import get_db
  5. import logging
  6. from sqlalchemy.exc import IntegrityError
  7. from utils import vectorizer
  8. from utils.vectorizer import Vectorizer
  9. from sqlalchemy import func
  10. from service.kg_prop_service import KGPropService
  11. from service.kg_edge_service import KGEdgeService
  12. logger = logging.getLogger(__name__)
  13. DISTANCE_THRESHOLD = 0.65
  14. DISTANCE_THRESHOLD2 = 0.3
  15. class KGNodeService:
  16. def __init__(self, db: Session):
  17. self.db = db
  18. _cache = {}
  19. def search_title_index(self, index: str, title: str, top_k: int = 3):
  20. cache_key = f"{index}:{title}:{top_k}"
  21. if cache_key in self._cache:
  22. return self._cache[cache_key]
  23. query_embedding = Vectorizer.get_embedding(title)
  24. db = next(get_db())
  25. # 执行向量搜索
  26. results = (
  27. db.query(
  28. KGNode.id,
  29. KGNode.name,
  30. KGNode.category,
  31. KGNode.embedding.l2_distance(query_embedding).label('distance')
  32. )
  33. .filter(KGNode.status == 0)
  34. #过滤掉version不等于'er'的节点
  35. .filter(KGNode.version != 'ER')
  36. .filter(KGNode.embedding.l2_distance(query_embedding) <= DISTANCE_THRESHOLD2)
  37. .order_by('distance').limit(top_k).all()
  38. )
  39. results = [
  40. {
  41. "id": node.id,
  42. "title": node.name,
  43. "text": node.category,
  44. "score": 2.0-node.distance
  45. }
  46. for node in results
  47. ]
  48. self._cache[cache_key] = results
  49. return results
  50. def paginated_search(self, search_params: dict) -> dict:
  51. load_props = search_params.get('load_props', False)
  52. prop_service = KGPropService(self.db)
  53. edge_service = KGEdgeService(self.db)
  54. keyword = search_params.get('keyword', '')
  55. category = search_params.get('category', '')
  56. page_no = search_params.get('pageNo', 1)
  57. distance = search_params.get('distance',DISTANCE_THRESHOLD)
  58. limit = search_params.get('limit', 10)
  59. if page_no < 1:
  60. page_no = 1
  61. if limit < 1:
  62. limit = 10
  63. embedding = Vectorizer.get_embedding(keyword)
  64. offset = (page_no - 1) * limit
  65. try:
  66. # 构建基础查询条件
  67. base_query = self.db.query(func.count(KGNode.id)).filter(
  68. KGNode.status == 0,
  69. KGNode.embedding.l2_distance(embedding) < distance
  70. )
  71. # 如果有category,则添加额外过滤条件
  72. if category:
  73. base_query = base_query.filter(KGNode.category == category)
  74. # 如果有knowledge_ids,则添加额外过滤条件
  75. if search_params.get('knowledge_ids'):
  76. total_count = base_query.filter(
  77. KGNode.version.in_(search_params['knowledge_ids'])
  78. ).scalar()
  79. else:
  80. total_count = base_query.scalar()
  81. query = self.db.query(
  82. KGNode.id,
  83. KGNode.name,
  84. KGNode.category,
  85. KGNode.embedding.l2_distance(embedding).label('distance')
  86. )
  87. query = query.filter(KGNode.status == 0)
  88. #category有值时,过滤掉category不等于category的节点
  89. if category:
  90. query = query.filter(KGNode.category == category)
  91. if search_params.get('knowledge_ids'):
  92. query = query.filter(KGNode.version.in_(search_params['knowledge_ids']))
  93. query = query.filter(KGNode.embedding.l2_distance(embedding) < distance)
  94. results = query.order_by('distance').offset(offset).limit(limit).all()
  95. #将results相同distance的category=疾病的放在前面
  96. results = sorted(results, key=lambda x: (x.distance, not x.category == '疾病'))
  97. return {
  98. 'records': [{
  99. 'id': r.id,
  100. 'name': r.name,
  101. 'category': r.category,
  102. 'props': prop_service.get_props_by_ref_id(r.id) if load_props else [],
  103. #'edges':edge_service.get_edges_by_nodes(r.id, r.id,False) if load_props else [],
  104. 'distance': round(r.distance, 3)
  105. } for r in results],
  106. 'pagination': {
  107. 'total': total_count,
  108. 'pageNo': page_no,
  109. 'limit': limit,
  110. 'totalPages': (total_count + limit - 1) // limit
  111. }
  112. }
  113. except Exception as e:
  114. logger.error(f"分页查询失败: {str(e)}")
  115. raise e
  116. def create_node(self, node_data: dict):
  117. try:
  118. existing = self.db.query(KGNode).filter(
  119. KGNode.name == node_data['name'],
  120. KGNode.category == node_data['category'],
  121. KGNode.version == node_data.get('version')
  122. ).first()
  123. if existing:
  124. raise ValueError("Node already exists")
  125. new_node = KGNode(**node_data)
  126. self.db.add(new_node)
  127. self.db.commit()
  128. return new_node
  129. except IntegrityError as e:
  130. self.db.rollback()
  131. logger.error(f"创建节点失败: {str(e)}")
  132. raise ValueError("Database integrity error")
  133. def get_node(self, node_id: int):
  134. if node_id is None:
  135. raise ValueError("Node ID is required")
  136. node = self.db.query(KGNode).filter(KGNode.id == node_id, KGNode.status == 0).first()
  137. if not node:
  138. raise ValueError("Node not found")
  139. return {
  140. 'id': node.id,
  141. 'name': node.name,
  142. 'category': node.category,
  143. 'version': node.version
  144. }
  145. def update_node(self, node_id: int, update_data: dict):
  146. node = self.db.query(KGNode).get(node_id)
  147. if not node:
  148. raise ValueError("Node not found")
  149. try:
  150. for key, value in update_data.items():
  151. setattr(node, key, value)
  152. self.db.commit()
  153. return node
  154. except Exception as e:
  155. self.db.rollback()
  156. logger.error(f"更新节点失败: {str(e)}")
  157. raise ValueError("Update failed")
  158. def delete_node(self, node_id: int):
  159. node = self.db.query(KGNode).get(node_id)
  160. if not node:
  161. raise ValueError("Node not found")
  162. try:
  163. self.db.delete(node)
  164. self.db.commit()
  165. return None
  166. except Exception as e:
  167. self.db.rollback()
  168. logger.error(f"删除节点失败: {str(e)}")
  169. raise ValueError("Delete failed")
  170. def batch_process_er_nodes(self):
  171. batch_size = 200
  172. offset = 0
  173. while True:
  174. try:
  175. nodes = self.db.query(KGNode).filter(
  176. #KGNode.version == 'ER',
  177. KGNode.embedding == None
  178. ).offset(offset).limit(batch_size).all()
  179. if not nodes:
  180. break
  181. updated_nodes = []
  182. for node in nodes:
  183. if not node.embedding:
  184. embedding = Vectorizer.get_embedding(node.name)
  185. node.embedding = embedding
  186. updated_nodes.append(node)
  187. if updated_nodes:
  188. self.db.commit()
  189. offset += batch_size
  190. except Exception as e:
  191. self.db.rollback()
  192. print(f"批量处理ER节点失败: {str(e)}")
  193. raise ValueError("Batch process failed")