9 changed files with 176 additions and 10 deletions
@ -0,0 +1,10 @@ |
|||
package com.project.milvus.application; |
|||
|
|||
import com.project.milvus.domain.dto.TitleVector; |
|||
|
|||
public interface MilvusApplicationService { |
|||
/** |
|||
* 题目入向量数据库 |
|||
*/ |
|||
void insertTitle(TitleVector title); |
|||
} |
|||
@ -0,0 +1,98 @@ |
|||
package com.project.milvus.application.impl; |
|||
|
|||
import cn.hutool.core.collection.CollectionUtil; |
|||
import com.project.base.config.CustomIdGenerator; |
|||
import com.project.base.domain.exception.MissingParameterException; |
|||
import com.project.milvus.application.MilvusApplicationService; |
|||
import com.project.milvus.domain.dto.TitleVector; |
|||
import com.project.milvus.domain.service.CheckMilvusDomainService; |
|||
import com.project.milvus.domain.service.MilvusDemoService; |
|||
import io.milvus.v2.service.vector.response.SearchResp; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.codec.digest.DigestUtils; |
|||
import org.redisson.api.RLock; |
|||
import org.redisson.api.RedissonClient; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.data.redis.core.StringRedisTemplate; |
|||
import org.springframework.stereotype.Service; |
|||
|
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Random; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
public class MilvusApplicationServiceImpl implements MilvusApplicationService { |
|||
@Autowired |
|||
private MilvusDemoService milvusDemoService; |
|||
@Autowired |
|||
private CheckMilvusDomainService checkMilvusDomainService; |
|||
@Autowired |
|||
private CustomIdGenerator customIdGenerator; |
|||
@Autowired |
|||
private RedissonClient redissonClient; |
|||
|
|||
|
|||
//相似度阈值
|
|||
private static final Float SIMILARITY_THRESHOLD = 0.8f; |
|||
private static final String LOCK_KEY = "lock:title:pointIds:"; |
|||
|
|||
@Override |
|||
public void insertTitle(TitleVector title) { |
|||
//参数校验
|
|||
checkMilvusDomainService.check(title); |
|||
|
|||
String lockKey = buildLockKey(title.getPointIdsList()); |
|||
RLock lock = redissonClient.getLock(lockKey); |
|||
|
|||
boolean locked = false; |
|||
int retry = 3; |
|||
try { |
|||
while (retry > 0 && !locked) { |
|||
// 等待 2s,拿到锁后10s自动释放
|
|||
locked = lock.tryLock(2, 10, TimeUnit.SECONDS); |
|||
if (!locked) { |
|||
//休眠等待重试
|
|||
Thread.sleep(100 + new Random().nextInt(100)); |
|||
} |
|||
} |
|||
|
|||
if (!locked) { |
|||
throw new RuntimeException("当前知识点正在处理,请稍后再试"); |
|||
} |
|||
|
|||
//比较相似度
|
|||
List<List<SearchResp.SearchResult>> query = milvusDemoService.query(title.getPointIdsList(), title.getTitleVectorList()); |
|||
if (CollectionUtil.isNotEmpty(query) && CollectionUtil.isNotEmpty(query.get(0))) { |
|||
SearchResp.SearchResult searchResult = query.get(0).get(0); |
|||
Float score = searchResult.getScore(); |
|||
if(score.compareTo(SIMILARITY_THRESHOLD) == 1){ |
|||
throw new MissingParameterException("题目相似度"+ score +",超过阈值"); |
|||
} |
|||
} |
|||
|
|||
//新增数据
|
|||
title.setId(customIdGenerator.nextId(title)); |
|||
milvusDemoService.insertRecord(title); |
|||
|
|||
}catch (InterruptedException e) { |
|||
Thread.currentThread().interrupt(); |
|||
throw new RuntimeException("获取锁被中断", e); |
|||
} finally { |
|||
if (locked && lock.isHeldByCurrentThread()) { |
|||
lock.unlock(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
/** |
|||
* 构建锁的key |
|||
*/ |
|||
private String buildLockKey(List<Long> poinIds){ |
|||
Collections.sort(poinIds); |
|||
String collect = poinIds.stream().map(String::valueOf).collect(Collectors.joining(",")); |
|||
return LOCK_KEY + DigestUtils.md5Hex(collect); |
|||
} |
|||
} |
|||
@ -1,4 +1,4 @@ |
|||
package com.project.milvus.domain; |
|||
package com.project.milvus.domain.dto; |
|||
|
|||
import com.google.gson.Gson; |
|||
import com.google.gson.JsonElement; |
|||
@ -0,0 +1,7 @@ |
|||
package com.project.milvus.domain.service; |
|||
|
|||
import com.project.milvus.domain.dto.TitleVector; |
|||
|
|||
public interface CheckMilvusDomainService { |
|||
void check(TitleVector title); |
|||
} |
|||
@ -1,6 +1,6 @@ |
|||
package com.project.milvus.service; |
|||
package com.project.milvus.domain.service; |
|||
|
|||
import com.project.milvus.domain.TitleVector; |
|||
import com.project.milvus.domain.dto.TitleVector; |
|||
import io.milvus.v2.service.vector.response.GetResp; |
|||
import io.milvus.v2.service.vector.response.SearchResp; |
|||
|
|||
@ -0,0 +1,33 @@ |
|||
package com.project.milvus.domain.service.impl; |
|||
|
|||
import cn.hutool.core.collection.CollectionUtil; |
|||
import com.project.base.domain.exception.MissingParameterException; |
|||
import com.project.milvus.domain.dto.TitleVector; |
|||
import com.project.milvus.domain.service.CheckMilvusDomainService; |
|||
import org.springframework.stereotype.Service; |
|||
|
|||
import java.util.List; |
|||
|
|||
/** |
|||
* 校验扩展 |
|||
*/ |
|||
@Service |
|||
public class CheckMilvusDomainServiceImpl implements CheckMilvusDomainService { |
|||
@Override |
|||
public void check(TitleVector title) { |
|||
if (title == null) { |
|||
throw new MissingParameterException("请求参数缺失或格式错误"); |
|||
} |
|||
|
|||
List<Long> pointIdsList = title.getPointIdsList(); |
|||
List<Float> titleVectorList = title.getTitleVectorList(); |
|||
|
|||
if (CollectionUtil.isEmpty(pointIdsList) || CollectionUtil.isEmpty(titleVectorList)) { |
|||
throw new MissingParameterException("请求参数缺失或格式错误"); |
|||
} |
|||
|
|||
if(titleVectorList.size() != 1024){ |
|||
throw new MissingParameterException("向量格式错误"); |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue