提交代码
This commit is contained in:
jiang
parent
d8a873995b
commit
0a831ebb42
|
|
@ -4,7 +4,7 @@ import org.springframework.cloud.openfeign.FeignClient;
|
|||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
|
||||
@FeignClient(name = "onlineAnnotationService", url = "${label-studio.url}")
|
||||
@FeignClient(name = "onlineAnnotationService")
|
||||
public interface OnlineAnnotationService {
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -44,6 +44,22 @@ public class DataSetTeamMemberController extends BaseController {
|
|||
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询标注人员列表
|
||||
*/
|
||||
@RequiresPermissions("dataCenter:member:list")
|
||||
@GetMapping("/getAllList")
|
||||
public AjaxResult getAllList()
|
||||
{
|
||||
try {
|
||||
List<DataSetTeamMemberEntity> list = dataSetTeamMemberService.selectTeamMemberList(null);
|
||||
return AjaxResult.success(list);
|
||||
}catch (Exception e){
|
||||
return AjaxResult.error();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 导出标注人员列表
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,28 +1,34 @@
|
|||
package com.bonus.ai.controller.dataset;
|
||||
package com.bonus.ai.controller;
|
||||
|
||||
import com.bonus.ai.domain.dataset.DataSetEntity;
|
||||
import com.bonus.ai.mapper.DataSetBasicFileMapper;
|
||||
import com.bonus.common.core.web.controller.BaseController;
|
||||
import com.bonus.ai.service.dataset.DatasetService;
|
||||
import com.bonus.common.core.web.domain.AjaxResult;
|
||||
import com.bonus.common.core.web.page.TableDataInfo;
|
||||
import com.bonus.common.security.utils.SecurityUtils;
|
||||
import com.bonus.system.api.domain.SysUser;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.validation.annotation.Validated;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import com.bonus.common.core.utils.StringUtils;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
/**
|
||||
* @author bonus
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/dataset")
|
||||
@Slf4j
|
||||
public class DatasetController extends BaseController {
|
||||
|
||||
@Resource
|
||||
@Autowired
|
||||
private DatasetService datasetService;
|
||||
|
||||
|
||||
/**创建数据集
|
||||
* @param dataSet 新增的数据集信息，并且建立和文件的关联关系
|
||||
* @return 返回影响的行数或错误信
|
||||
|
|
@ -64,17 +70,20 @@ public class DatasetController extends BaseController {
|
|||
}
|
||||
|
||||
/**
|
||||
* 查看由我创建的数据集和公共数据集
|
||||
* 查看由我创建的数据集
|
||||
* 统一获取文件列表
|
||||
* @param type 参数类型：my - 当前用户创建的文件, public - 公共文件, all - 用户创建和公共文件
|
||||
* @param
|
||||
* @return 返回满足条件的数据集列表
|
||||
*/
|
||||
@GetMapping("/list")
|
||||
public AjaxResult list(@RequestParam String type) {
|
||||
AjaxResult ajax = AjaxResult.success();
|
||||
List<DataSetEntity> datasetList = datasetService.getDatasetList(type);
|
||||
ajax.put(AjaxResult.DATA_TAG, datasetList);
|
||||
return ajax;
|
||||
public TableDataInfo list(DataSetEntity dataSet) {
|
||||
try {
|
||||
startPage();
|
||||
List<DataSetEntity> datasetList = datasetService.getDatasetList(dataSet);
|
||||
return getDataTable(datasetList);
|
||||
}catch (Exception e){
|
||||
return getDataTable(new ArrayList<>());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -27,7 +27,10 @@ public class DataSetEntity extends BaseEntity {
|
|||
private String datasetDesc;
|
||||
/** 数据类型:0图片,1视频 ,2音频，3自由格式*/
|
||||
private String dataType;
|
||||
|
||||
private String inputPath;
|
||||
private Long inputId;
|
||||
private Long outputId;
|
||||
private String outputPath;
|
||||
|
||||
/** 数据集来源，0 本地上传，1 minio上传*/
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ public interface DatasetFileMapper {
|
|||
* @param datasetIds 数据集ID
|
||||
* @return 删除的行数
|
||||
*/
|
||||
int deleteDatasetFiles(@Param("datasetIds") Long[] datasetIds);
|
||||
int deleteDatasetFiles(Long[] datasetIds);
|
||||
//
|
||||
/**
|
||||
* 根据数据集ID查询文件列表
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ public interface DatasetMapper {
|
|||
/**
|
||||
* 根据ID批量删除数据集
|
||||
*/
|
||||
int deleteByIds(@Param("datasetIds") Long[] datasetIds);
|
||||
int deleteByIds( Long[] datasetIds);
|
||||
|
||||
/**
|
||||
* 根据ID查询数据集
|
||||
|
|
@ -39,22 +39,8 @@ public interface DatasetMapper {
|
|||
/**
|
||||
* 查询数据集列表
|
||||
*/
|
||||
List<DataSetEntity> selectList(DataSetEntity dataset);
|
||||
List<DataSetEntity> getDatasetList(DataSetEntity dataset);
|
||||
|
||||
/**
|
||||
* 查询公开的数据集列表
|
||||
*/
|
||||
List<DataSetEntity> selectPublicList();
|
||||
|
||||
/**
|
||||
* 查询公开的数据集列表
|
||||
*/
|
||||
List<DataSetEntity> selectMyList(String userName);
|
||||
|
||||
/**
|
||||
* 查询由我创建的和公开的数据集列表
|
||||
*/
|
||||
List<DataSetEntity> selectAllList(String userName);
|
||||
|
||||
/**
|
||||
* 更新数据集公开状态
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
package com.bonus.ai.service.Impl.dataset;
|
||||
|
||||
import com.bonus.ai.config.OnlineAnnotateConfig;
|
||||
import com.bonus.ai.domain.DataSetBasicFileEntity;
|
||||
import com.bonus.ai.domain.dataset.DataSetEntity;
|
||||
import com.bonus.ai.domain.dataset.DatasetFile;
|
||||
import com.bonus.ai.mapper.DataSetBasicFileMapper;
|
||||
import com.bonus.ai.mapper.DatasetFileMapper;
|
||||
import com.bonus.ai.mapper.DatasetMapper;
|
||||
import com.bonus.ai.service.dataset.DatasetService;
|
||||
|
|
@ -11,12 +13,17 @@ import com.bonus.common.core.utils.StringUtils;
|
|||
import com.bonus.common.security.utils.SecurityUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import com.bonus.common.security.utils.SecurityUtils;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class DatasetServiceImpl implements DatasetService {
|
||||
|
|
@ -31,10 +38,19 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
@Resource
|
||||
private DatasetFileMapper datasetFileMapper;
|
||||
|
||||
@Resource
|
||||
OnlineAnnotateConfig onlineAnnotateConfig;
|
||||
|
||||
@Resource
|
||||
private DataSetBasicFileMapper dataSetBasicFileMapper;
|
||||
|
||||
@Override
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
public int createDataset(DataSetEntity dataSet) {
|
||||
try {
|
||||
dataSet.setCreateBy(SecurityUtils.getUserId().toString());
|
||||
dataSet.setUpdateBy(SecurityUtils.getUserId().toString());
|
||||
dataSet.setDatasetUuid(StringUtils.randomUUID());
|
||||
int rows = datasetMapper.insert(dataSet);
|
||||
// 新增数据集与文件管理
|
||||
insertDatasetFileMap(dataSet);
|
||||
|
|
@ -74,17 +90,11 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<DataSetEntity> getDatasetList(String type) {
|
||||
public List<DataSetEntity> getDatasetList(DataSetEntity dataSet) {
|
||||
try {
|
||||
String userName = SecurityUtils.getUsername();
|
||||
if (StringUtils.isEmpty(type) || LISTTYPE_ALL.equals(type)){
|
||||
return datasetMapper.selectAllList(userName);
|
||||
}else if (LISTTYPE_PUBLIC.equals(type)){
|
||||
datasetMapper.selectPublicList();
|
||||
}else {
|
||||
datasetMapper.selectMyList(userName);
|
||||
}
|
||||
return new ArrayList<>();
|
||||
String userId = SecurityUtils.getUserId().toString();
|
||||
dataSet.setCreateBy(userId);
|
||||
return datasetMapper.getDatasetList(dataSet);
|
||||
} catch (Exception e) {
|
||||
log.error("获取数据集列表失败", e);
|
||||
return new ArrayList<>();
|
||||
|
|
@ -125,21 +135,82 @@ public class DatasetServiceImpl implements DatasetService {
|
|||
return UserConstants.UNIQUE;
|
||||
}
|
||||
|
||||
private void insertDatasetFileMap(DataSetEntity dataSet){
|
||||
this.insertDatasetFileMap(dataSet.getDatasetId(), dataSet.getFileIds(),dataSet.getIsAnnotated());
|
||||
/**
|
||||
* 插入数据集文件映射关系
|
||||
* @param dataSet 数据集实体，包含数据集ID、输入ID、数据类型等信息
|
||||
*/
|
||||
private void insertDatasetFileMap(DataSetEntity dataSet) {
|
||||
// 根据父目录ID查询其下的文件和文件夹列表
|
||||
List<DataSetBasicFileEntity> fileList = dataSetBasicFileMapper.selectDataSetBasicFileByParentId(dataSet.getInputId());
|
||||
|
||||
// 根据数据类型获取支持的文件后缀列表，例如图片、视频或音频
|
||||
List<String> supportedFormats = getSupportedFormats(dataSet.getDataType());
|
||||
|
||||
// 筛选符合条件的文件ID列表
|
||||
List<Long> fileIdList = fileList.stream()
|
||||
.filter(file -> isValidFile(file, supportedFormats)) // 过滤出符合条件的文件
|
||||
.map(DataSetBasicFileEntity::getFileId) // 提取文件ID
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// 批量插入文件映射关系
|
||||
insertDatasetFileMap(dataSet.getDatasetId(), fileIdList, dataSet.getIsAnnotated());
|
||||
}
|
||||
|
||||
private void insertDatasetFileMap(Long datasetId, Long[] fileIds, String isAnnotated){
|
||||
if (StringUtils.isNotEmpty(fileIds)){
|
||||
List <DatasetFile> list = new ArrayList<DatasetFile>();
|
||||
for (Long fileId : fileIds){
|
||||
DatasetFile datasetFile = new DatasetFile();
|
||||
datasetFile.setDatasetId(datasetId);
|
||||
datasetFile.setFileId(fileId);
|
||||
datasetFile.setIsAnnotated(isAnnotated);
|
||||
list.add(datasetFile);
|
||||
}
|
||||
/**
|
||||
* 根据数据类型获取支持的文件后缀列表
|
||||
* @param dataType 数据类型：0 - 图片，1 - 视频，2 - 音频
|
||||
* @return 支持的文件后缀列表，例如图片格式返回 [".jpg", ".png", ".jpeg", ".bmp"]
|
||||
*/
|
||||
private List<String> getSupportedFormats(String dataType) {
|
||||
switch (dataType) {
|
||||
case "0": // 图片类型
|
||||
return Arrays.asList(".jpg", ".png", ".jpeg", ".bmp");
|
||||
case "1": // 视频类型
|
||||
return Collections.singletonList(".mp4");
|
||||
case "2": // 音频类型
|
||||
return Collections.singletonList(".wav");
|
||||
default: // 未知类型返回空列表，避免空指针异常
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断文件是否符合筛选条件
|
||||
* @param file 数据集基础文件实体，包含文件名、是否为目录等属性
|
||||
* @param supportedFormats 支持的文件后缀列表
|
||||
* @return 如果文件符合条件则返回 true，否则返回 false
|
||||
*/
|
||||
private boolean isValidFile(DataSetBasicFileEntity file, List<String> supportedFormats) {
|
||||
return "0".equals(file.getIsDirectory()) && // 确保文件不是目录
|
||||
supportedFormats.stream() // 遍历支持的文件后缀
|
||||
.anyMatch(format -> file.getFileName().toLowerCase().endsWith(format)); // 文件名后缀匹配
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量插入文件映射关系
|
||||
* @param datasetId 数据集ID
|
||||
* @param fileIdList 文件ID列表
|
||||
* @param isAnnotated 是否标注标志
|
||||
*/
|
||||
private void insertDatasetFileMap(Long datasetId, List<Long> fileIdList, String isAnnotated) {
|
||||
// 确保文件列表非空
|
||||
if (fileIdList != null && !fileIdList.isEmpty()) {
|
||||
// 使用流构造 DatasetFile 对象列表
|
||||
List<DatasetFile> datasetFileList = fileIdList.stream()
|
||||
// 为每个文件ID创建 DatasetFile 对象
|
||||
.map(fileId -> {
|
||||
DatasetFile datasetFile = new DatasetFile();
|
||||
datasetFile.setDatasetId(datasetId); // 设置数据集ID
|
||||
datasetFile.setFileId(fileId); // 设置文件ID
|
||||
datasetFile.setIsAnnotated(isAnnotated); // 设置是否标注
|
||||
return datasetFile;
|
||||
})
|
||||
// 收集为列表
|
||||
.collect(Collectors.toList());
|
||||
|
||||
// 调用 Mapper 批量插入映射关系
|
||||
datasetFileMapper.batchDatasetFile(datasetFileList);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -28,10 +28,10 @@ public interface DatasetService {
|
|||
|
||||
/**
|
||||
* 获取数据集列表
|
||||
* @param type 类型：my/public/all
|
||||
* @param dataSet 实体
|
||||
* @return 数据集列表
|
||||
*/
|
||||
List<DataSetEntity> getDatasetList(String type);
|
||||
List<DataSetEntity> getDatasetList(DataSetEntity dataSet);
|
||||
|
||||
/**
|
||||
* 获取数据集详情
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@
|
|||
abf.file_size, abf.file_last_modifytime, abf.upload_time, abf.is_directory, abf.is_public,
|
||||
abf.del_flag, su.user_name AS create_by , abf.create_time, abf.update_by, abf.update_time
|
||||
from ai_basic_file abf
|
||||
LEFT JOIN sys_user su ON abf.create_by = su.user_id
|
||||
LEFT JOIN sys_user su ON abf.create_by = su.user_id AND su.del_flag ='0'
|
||||
|
||||
</sql>
|
||||
|
||||
|
|
@ -41,13 +41,13 @@
|
|||
|
||||
<select id="selectDataSetBasicFileByFileId" parameterType="Long" resultMap="DataSetBasicFileResult">
|
||||
<include refid="selectDataSetBasicFileVo"/>
|
||||
where abf.file_id = #{fileId}
|
||||
where abf.file_id = #{fileId} AND abf.del_flag ='0'
|
||||
ORDER BY abf.is_directory DESC ,abf.create_time DESC
|
||||
</select>
|
||||
|
||||
<select id="selectDataSetBasicFileByParentId" resultMap="DataSetBasicFileResult">
|
||||
<include refid="selectDataSetBasicFileVo"/>
|
||||
where abf.parent_id = #{parentId}
|
||||
where abf.parent_id = #{parentId} AND abf.del_flag ='0'
|
||||
ORDER BY abf.is_directory DESC ,abf.create_time DESC
|
||||
</select>
|
||||
|
||||
|
|
|
|||
|
|
@ -27,8 +27,9 @@
|
|||
</delete>
|
||||
|
||||
<!-- 删除数据集文件映射关系 -->
|
||||
<delete id="deleteDatasetFiles">
|
||||
DELETE FROM ai_dataset_file_map
|
||||
<delete id="deleteDatasetFiles" parameterType="String">
|
||||
delete from ai_dataset_file_map
|
||||
where dataset_id in
|
||||
<foreach collection="array" item="datasetId" open="(" separator="," close=")">
|
||||
#{datasetId}
|
||||
</foreach>
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@
|
|||
<result property="dataSource" column="data_source"/>
|
||||
<result property="inputPath" column="input_path"/>
|
||||
<result property="outputPath" column="output_path"/>
|
||||
<result property="inputId" column="input_id"/>
|
||||
<result property="outputId" column="output_id"/>
|
||||
<result property="annotationStatus" column="annotation_status"/>
|
||||
<result property="isPublic" column="is_public"/>
|
||||
<result property="delFlag" column="del_flag"/>
|
||||
|
|
@ -24,19 +26,17 @@
|
|||
|
||||
<sql id="selectDatasetVo">
|
||||
select dataset_id, dataset_uuid, dataset_name, description, data_type, data_source, input_path,
|
||||
output_path, annotation_status, is_annotated, is_public, del_flag, create_by, create_time,
|
||||
output_path, annotation_status, is_public, del_flag, create_by, create_time,
|
||||
update_by, update_time
|
||||
from ai_dataset
|
||||
</sql>
|
||||
|
||||
<insert id="insert" parameterType="com.bonus.ai.domain.dataset.DataSetEntity" useGeneratedKeys="true" keyProperty="datasetId">
|
||||
insert into ai_dataset (
|
||||
dataset_uuid, dataset_name, description, data_type, data_source, input_path, output_path,
|
||||
annotation_status, is_annotated, is_public, project_id, del_flag, create_by, create_time, update_by, update_time
|
||||
dataset_uuid, dataset_name, description, data_type, data_source, input_path, input_id, output_path,output_id, create_by, update_by
|
||||
) values (
|
||||
#{datasetUuid}, #{datasetName}, #{datasetDesc}, #{dataType},#{dataSource}, #{inputPath},
|
||||
#{outputPath}, #{annotationStatus}, #{isAnnotated}, #{isPublic}, #{delFlag}, #{createBy},
|
||||
#{createTime}, #{updateBy}, #{updateTime}
|
||||
#{datasetUuid}, #{datasetName}, #{datasetDesc}, #{dataType},#{dataSource}, #{inputPath},#{inputId},
|
||||
#{outputPath},#{outputId}, #{createBy}, #{updateBy}
|
||||
)
|
||||
</insert>
|
||||
|
||||
|
|
@ -63,10 +63,10 @@
|
|||
where dataset_id = #{datasetId}
|
||||
</delete>
|
||||
|
||||
<delete id="deleteByIds">
|
||||
<delete id="deleteByIds" parameterType="String">
|
||||
update ai_dataset set del_flag = '1'
|
||||
where dataset_id in
|
||||
<foreach collection="datasetIds" item="datasetId" open="(" separator="," close=")">
|
||||
<foreach collection="array" item="datasetId" open="(" separator="," close=")">
|
||||
#{datasetId}
|
||||
</foreach>
|
||||
</delete>
|
||||
|
|
@ -88,7 +88,7 @@
|
|||
and del_flag = '0'
|
||||
</select>
|
||||
|
||||
<select id="selectList" parameterType="com.bonus.ai.domain.dataset.DataSetEntity" resultMap="DatasetResult">
|
||||
<select id="getDatasetList" parameterType="com.bonus.ai.domain.dataset.DataSetEntity" resultMap="DatasetResult">
|
||||
<include refid="selectDatasetVo"/>
|
||||
<where>
|
||||
and del_flag = '0'
|
||||
|
|
@ -111,27 +111,6 @@
|
|||
order by create_time desc
|
||||
</select>
|
||||
|
||||
<select id="selectPublicList" resultMap="DatasetResult">
|
||||
<include refid="selectDatasetVo"/>
|
||||
where is_public = '1'
|
||||
and del_flag = '0'
|
||||
order by create_time desc
|
||||
</select>
|
||||
|
||||
<select id="selectMyList" resultMap="DatasetResult">
|
||||
<include refid="selectDatasetVo"/>
|
||||
where del_flag = '0'
|
||||
and create_by = #{createBy}
|
||||
order by create_time desc
|
||||
</select>
|
||||
|
||||
<select id="selectAllList" resultMap="DatasetResult">
|
||||
<include refid="selectDatasetVo"/>
|
||||
where del_flag = '0'
|
||||
and (create_by = #{createBy} or is_public = '1')
|
||||
order by create_time desc
|
||||
</select>
|
||||
|
||||
<update id="updatePublicStatus">
|
||||
update ai_dataset
|
||||
set is_public = #{isPublic},
|
||||
|
|
@ -152,5 +131,8 @@
|
|||
where dataset_name = #{datasetName}
|
||||
and del_flag = '0' limit 1
|
||||
</select>
|
||||
<select id="selectAllList" resultMap="DatasetResult">
|
||||
|
||||
</select>
|
||||
|
||||
</mapper>
|
||||
Loading…
Reference in New Issue