|
|
@ -8,14 +8,18 @@ import com.project.milvus.application.MilvusApplicationService; |
|
|
import com.project.milvus.domain.dto.TitleVector; |
|
|
import com.project.milvus.domain.dto.TitleVector; |
|
|
import com.project.milvus.domain.service.CheckMilvusDomainService; |
|
|
import com.project.milvus.domain.service.CheckMilvusDomainService; |
|
|
import com.project.milvus.domain.service.MilvusDemoService; |
|
|
import com.project.milvus.domain.service.MilvusDemoService; |
|
|
|
|
|
import com.project.question.domain.dto.QuestionDTO; |
|
|
|
|
|
import com.project.question.domain.entity.QuestionEntity; |
|
|
|
|
|
import com.project.question.domain.service.QuestionBaseService; |
|
|
import io.milvus.v2.service.vector.response.SearchResp; |
|
|
import io.milvus.v2.service.vector.response.SearchResp; |
|
|
import lombok.extern.slf4j.Slf4j; |
|
|
import lombok.extern.slf4j.Slf4j; |
|
|
import org.apache.commons.codec.digest.DigestUtils; |
|
|
import org.apache.commons.codec.digest.DigestUtils; |
|
|
import org.redisson.api.RLock; |
|
|
import org.redisson.api.RLock; |
|
|
import org.redisson.api.RedissonClient; |
|
|
import org.redisson.api.RedissonClient; |
|
|
|
|
|
import org.springframework.beans.BeanUtils; |
|
|
import org.springframework.beans.factory.annotation.Autowired; |
|
|
import org.springframework.beans.factory.annotation.Autowired; |
|
|
import org.springframework.stereotype.Service; |
|
|
import org.springframework.stereotype.Service; |
|
|
import org.springframework.web.client.RestTemplate; |
|
|
import org.springframework.transaction.annotation.Transactional; |
|
|
|
|
|
|
|
|
import java.util.*; |
|
|
import java.util.*; |
|
|
import java.util.concurrent.TimeUnit; |
|
|
import java.util.concurrent.TimeUnit; |
|
|
@ -29,9 +33,9 @@ public class MilvusApplicationServiceImpl implements MilvusApplicationService { |
|
|
@Autowired |
|
|
@Autowired |
|
|
private CheckMilvusDomainService checkMilvusDomainService; |
|
|
private CheckMilvusDomainService checkMilvusDomainService; |
|
|
@Autowired |
|
|
@Autowired |
|
|
private CustomIdGenerator customIdGenerator; |
|
|
|
|
|
@Autowired |
|
|
|
|
|
private RedissonClient redissonClient; |
|
|
private RedissonClient redissonClient; |
|
|
|
|
|
@Autowired |
|
|
|
|
|
private QuestionBaseService questionBaseService; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
//相似度阈值
|
|
|
//相似度阈值
|
|
|
@ -39,22 +43,25 @@ public class MilvusApplicationServiceImpl implements MilvusApplicationService { |
|
|
private static final String LOCK_KEY = "lock:title:pointIds:"; |
|
|
private static final String LOCK_KEY = "lock:title:pointIds:"; |
|
|
|
|
|
|
|
|
@Override |
|
|
@Override |
|
|
|
|
|
@Transactional(rollbackFor = Exception.class) |
|
|
public void insertTitle(TitleVector title) { |
|
|
public void insertTitle(TitleVector title) { |
|
|
//参数校验
|
|
|
// 1. 基础参数校验(在锁外,快速失败)
|
|
|
checkMilvusDomainService.check(title); |
|
|
checkMilvusDomainService.checkBasic(title); |
|
|
|
|
|
|
|
|
String lockKey = buildLockKey(title.getPointIdsList()); |
|
|
String lockKey = buildLockKey(title.getPointIdsList()); |
|
|
RLock lock = redissonClient.getLock(lockKey); |
|
|
RLock lock = redissonClient.getLock(lockKey); |
|
|
|
|
|
|
|
|
boolean locked = false; |
|
|
boolean locked = false; |
|
|
|
|
|
//重试 3 次
|
|
|
int retry = 3; |
|
|
int retry = 3; |
|
|
try { |
|
|
try { |
|
|
while (retry > 0 && !locked) { |
|
|
while (retry > 0 && !locked) { |
|
|
// 等待 2s,拿到锁后10s自动释放
|
|
|
// 等待 2s,拿到锁后 10s 自动释放
|
|
|
locked = lock.tryLock(2, 10, TimeUnit.SECONDS); |
|
|
locked = lock.tryLock(2, 10, TimeUnit.SECONDS); |
|
|
if (!locked) { |
|
|
if (!locked) { |
|
|
//休眠等待重试
|
|
|
//休眠等待重试
|
|
|
Thread.sleep(100 + new Random().nextInt(100)); |
|
|
Thread.sleep(100 + new Random().nextInt(100)); |
|
|
|
|
|
retry--; |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
@ -62,18 +69,25 @@ public class MilvusApplicationServiceImpl implements MilvusApplicationService { |
|
|
throw new RuntimeException("当前知识点正在处理,请稍后再试"); |
|
|
throw new RuntimeException("当前知识点正在处理,请稍后再试"); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
//唯一性校验
|
|
|
|
|
|
checkMilvusDomainService.checkUnique(title); |
|
|
|
|
|
|
|
|
//比较相似度
|
|
|
//比较相似度
|
|
|
List<List<SearchResp.SearchResult>> query = milvusDemoService.query(title); |
|
|
List<List<SearchResp.SearchResult>> query = milvusDemoService.query(title); |
|
|
if (CollectionUtil.isNotEmpty(query) && CollectionUtil.isNotEmpty(query.get(0))) { |
|
|
if (CollectionUtil.isNotEmpty(query) && CollectionUtil.isNotEmpty(query.get(0))) { |
|
|
SearchResp.SearchResult searchResult = query.get(0).get(0); |
|
|
SearchResp.SearchResult searchResult = query.get(0).get(0); |
|
|
Float score = searchResult.getScore(); |
|
|
Float score = searchResult.getScore(); |
|
|
if(score.compareTo(SIMILARITY_THRESHOLD) == 1){ |
|
|
if(score.compareTo(SIMILARITY_THRESHOLD) > 0){ |
|
|
throw new MissingParameterException("题目相似度"+ score +",超过阈值"); |
|
|
throw new MissingParameterException("题目相似度"+ score +",超过阈值"); |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
//新增数据
|
|
|
//保存题目到数据库
|
|
|
title.setId(customIdGenerator.nextId(title)); |
|
|
QuestionEntity questionEntity = buildQuestionEntity(title); |
|
|
|
|
|
questionBaseService.save(questionEntity); |
|
|
|
|
|
|
|
|
|
|
|
//保存向量数据到 Milvus
|
|
|
|
|
|
title.setId(questionEntity.getId()); |
|
|
milvusDemoService.insertRecord(title); |
|
|
milvusDemoService.insertRecord(title); |
|
|
|
|
|
|
|
|
}catch (InterruptedException e) { |
|
|
}catch (InterruptedException e) { |
|
|
@ -87,7 +101,26 @@ public class MilvusApplicationServiceImpl implements MilvusApplicationService { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
* 构建锁的key |
|
|
* 构建题目实体 |
|
|
|
|
|
*/ |
|
|
|
|
|
private QuestionEntity buildQuestionEntity(TitleVector title) { |
|
|
|
|
|
QuestionEntity questionEntity = new QuestionEntity(); |
|
|
|
|
|
questionEntity.setSourceType(0); // 0-单 KP 出题
|
|
|
|
|
|
questionEntity.setUseStatus(false); // 未使用
|
|
|
|
|
|
questionEntity.setKpIdList(title.getPointIdsList()); |
|
|
|
|
|
questionEntity.setUniqueHash(title.getUniqueHash()); |
|
|
|
|
|
questionEntity.setQuestionType(title.QuestionDetailDTO.getType()); |
|
|
|
|
|
|
|
|
|
|
|
// 构建题目详情
|
|
|
|
|
|
QuestionEntity.QuestionDetail detail = new QuestionEntity.QuestionDetail(); |
|
|
|
|
|
BeanUtils.copyProperties(title.QuestionDetailDTO, detail); |
|
|
|
|
|
questionEntity.setQuestionDetail(detail); |
|
|
|
|
|
|
|
|
|
|
|
return questionEntity; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* 构建锁的 key |
|
|
*/ |
|
|
*/ |
|
|
private String buildLockKey(List<Long> poinIds){ |
|
|
private String buildLockKey(List<Long> poinIds){ |
|
|
String collect = poinIds.stream().map(String::valueOf).collect(Collectors.joining(",")); |
|
|
String collect = poinIds.stream().map(String::valueOf).collect(Collectors.joining(",")); |
|
|
|