luoweijian
/
ai-evaluator
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

导出id精度缺失问题

master
luogw 2 weeks ago
parent
7aee0c9985
commit
30f8c39fe1
2 changed files with 69 additions and 47 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 6
      src/main/java/com/project/information/application/impl/KnowledgePointApplicationServiceImpl.java
  2. 110
      src/main/java/com/project/information/domain/service/impl/UploadInformationDomainServiceImpl.java

6
src/main/java/com/project/information/application/impl/KnowledgePointApplicationServiceImpl.java
View File

@ -84,8 +84,6 @@ public class KnowledgePointApplicationServiceImpl implements KnowledgePointAppli
try (InputStream inputStream = minIoUtils.getObject(path)) {
uploadToPython(inputStream, filename , id);
}
// 成功后更新状态
updateStatusInformation(null, InformationParseStatusEnum.InProgress.getValue(), id);
} catch (Exception e) {
log.error("解析异常, ID:{}", id, e);
@ -143,11 +141,11 @@ public class KnowledgePointApplicationServiceImpl implements KnowledgePointAppli
if (!CollectionUtils.isEmpty(knowledgePointEntities)) {
knowledgePointBaseService.saveBatch(knowledgePointEntities);
// 更新资料状态为：成功
updateStatusInformation(knowledgePointEntities.get(0).getParseName(), InformationParseStatusEnum.InProgress.getValue(), id);
updateStatusInformation(knowledgePointEntities.get(0).getParseName(), InformationParseStatusEnum.Success.getValue(), id);
} else {
log.warn("文件 [{}] 解析成功但未提取到知识点", fileName);
// 更新资料状态为：成功 (或根据业务定义为无知识点状态)
updateStatusInformation(null, InformationParseStatusEnum.InProgress.getValue(), id);
updateStatusInformation(null, InformationParseStatusEnum.Success.getValue(), id);
}
}
/**

110
src/main/java/com/project/information/domain/service/impl/UploadInformationDomainServiceImpl.java
View File

@ -5,16 +5,13 @@ import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.file.FileNameUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.db.handler.StringHandler;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.project.base.config.CustomIdGenerator;
import com.project.base.domain.exception.BusinessErrorException;
import com.project.base.domain.result.Result;
import com.project.information.application.KnowledgePointApplicationService;
import com.project.information.domain.dto.InformationDTO;
import com.project.information.domain.dto.KnowledgePointDTO;
import com.project.information.domain.entity.InformationEntity;
import com.project.information.domain.entity.KnowledgePointEntity;
import com.project.information.domain.entity.ProductLineEntity;
import com.project.information.domain.enums.InformationParseStatusEnum;
import com.project.information.domain.param.FileCheckItem;
@ -29,12 +26,14 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.io.InputStream;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
@Service
public class UploadInformationDomainServiceImpl implements UploadInformationDomainService {
private final ConcurrentHashMap<Long, ReentrantLock> subLineUploadLocks = new ConcurrentHashMap<>();
@Autowired
private InformationBaseService informationBaseService;
@ -108,49 +107,74 @@ public class UploadInformationDomainServiceImpl implements UploadInformationDoma
}
List<String> successFiles = new ArrayList<>();
Map<Long, String> fileMap = new HashMap<>();
Map<Long, String> fileNameMap = new HashMap<>();
for (MultipartFile file : files) {
String fileName = file.getOriginalFilename();
ReentrantLock lock = subLineUploadLocks.computeIfAbsent(subLineId, key -> new ReentrantLock());
lock.lock();
try {
List<MultipartFile> sortedFiles = Arrays.stream(files)
.sorted(Comparator.comparing(file -> Optional.ofNullable(file.getOriginalFilename()).orElse("")))
.toList();
List<String> inputFileNames = sortedFiles.stream()
.map(MultipartFile::getOriginalFilename)
.filter(Objects::nonNull)
.toList();
Set<String> existNameSet = new HashSet<>();
if (CollUtil.isNotEmpty(inputFileNames)) {
existNameSet = informationBaseService.list(new LambdaQueryWrapper<InformationEntity>()
.select(InformationEntity::getName)
.eq(InformationEntity::getSubLineId, subLineId)
.in(InformationEntity::getName, inputFileNames))
.stream()
.map(InformationEntity::getName)
.collect(java.util.stream.Collectors.toSet());
}
LambdaQueryWrapper<InformationEntity> wrapper = new LambdaQueryWrapper<InformationEntity>()
.eq(InformationEntity::getSubLineId, subLineId)
.eq(InformationEntity::getName, fileName);
long count = informationBaseService.count(wrapper);
//如果存在重复的，并且是跳过则不做处理
if (count > 0 && skipExisting) {
continue;
List<String> successFiles = new ArrayList<>();
Map<Long, String> fileMap = new HashMap<>();
Map<Long, String> fileNameMap = new HashMap<>();
for (MultipartFile file : sortedFiles) {
String fileName = file.getOriginalFilename();
if (Objects.isNull(fileName)) {
continue;
}
if (existNameSet.contains(fileName) && skipExisting) {
continue;
}
LambdaQueryWrapper<InformationEntity> wrapper = new LambdaQueryWrapper<InformationEntity>()
.eq(InformationEntity::getSubLineId, subLineId)
.eq(InformationEntity::getName, fileName);
informationBaseService.remove(wrapper);
String filePath = String.format("%s/%s/%s.%s",
subLineId,
DateUtil.format(new Date(), "yyyyMMdd"),
IdUtil.fastSimpleUUID(),
FileNameUtil.extName(fileName));
minIoUtils.uploadFile(file.getInputStream() , filePath);
successFiles.add(fileName);
InformationDTO informationDTO = new InformationDTO();
informationDTO.setSubLineId(subLineId);
informationDTO.setSubLineName(productLine.getName());
informationDTO.setName(fileName);
informationDTO.setFileSuffix(FileUtil.getSuffix(fileName));
informationDTO.setFilePath(filePath);
informationDTO.setParseStatus(InformationParseStatusEnum.NotStart.getValue());
InformationEntity entity = informationDTO.toEntity(InformationEntity::new);
entity.setId(customIdGenerator.nextId(entity));
informationBaseService.save(entity);
//收集file
fileMap.put(entity.getId(), filePath);
fileNameMap.put(entity.getId(), fileName);
}
// 删掉原来的
informationBaseService.remove(wrapper);
String filePath = String.format("%s/%s/%s.%s",
subLineId,
DateUtil.format(new Date(), "yyyyMMdd"),
IdUtil.fastSimpleUUID(),
FileNameUtil.extName(fileName));
minIoUtils.uploadFile(file.getInputStream() , filePath);
successFiles.add(fileName);
InformationDTO informationDTO = new InformationDTO();
informationDTO.setSubLineId(subLineId);
informationDTO.setSubLineName(productLine.getName());
informationDTO.setName(fileName);
informationDTO.setFileSuffix(FileUtil.getSuffix(fileName));
informationDTO.setFilePath(filePath);
informationDTO.setParseStatus(InformationParseStatusEnum.NotStart.getValue());
InformationEntity entity = informationDTO.toEntity(InformationEntity::new);
entity.setId(customIdGenerator.nextId(entity));
informationBaseService.save(entity);
//收集file
fileMap.put(entity.getId(), filePath);
fileNameMap.put(entity.getId(), fileName);
}
//发起解析文档知识点
knowledgePointApplicationService.parse(fileMap,fileNameMap);
//发起解析文档知识点
knowledgePointApplicationService.parse(fileMap,fileNameMap);
return Result.success(String.format("上传成功：【%s】" , String.join("," , successFiles)));
return Result.success(String.format("上传成功：【%s】" , String.join("," , successFiles)));
} finally {
lock.unlock();
}
}
}

Write Preview
Loading…
Cancel
Save