fix(personal): isolate personal objects and dedupe under lock
This commit is contained in:
+170
-142
@@ -1,5 +1,6 @@
|
|||||||
package org.dromara.aihr.personal.service;
|
package org.dromara.aihr.personal.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -8,12 +9,13 @@ import org.dromara.aihr.personal.domain.PersonalAssistantDto.TextItemRequest;
|
|||||||
import org.dromara.aihr.personal.support.PersonalKnowledgeProperties;
|
import org.dromara.aihr.personal.support.PersonalKnowledgeProperties;
|
||||||
import org.dromara.aihr.personal.support.PersonalOwner;
|
import org.dromara.aihr.personal.support.PersonalOwner;
|
||||||
import org.dromara.common.core.exception.ServiceException;
|
import org.dromara.common.core.exception.ServiceException;
|
||||||
|
import org.dromara.common.oss.core.OssClient;
|
||||||
|
import org.dromara.common.oss.entity.UploadResult;
|
||||||
import org.dromara.common.oss.factory.OssFactory;
|
import org.dromara.common.oss.factory.OssFactory;
|
||||||
import org.dromara.system.domain.vo.SysOssVo;
|
import org.dromara.system.domain.vo.SysOssVo;
|
||||||
import org.dromara.system.service.ISysOssService;
|
import org.dromara.system.service.ISysOssService;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.jdbc.core.JdbcTemplate;
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
import org.springframework.jdbc.support.GeneratedKeyHolder;
|
|
||||||
import org.springframework.jdbc.support.KeyHolder;
|
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
import org.springframework.transaction.annotation.Transactional;
|
||||||
import org.springframework.transaction.support.TransactionSynchronization;
|
import org.springframework.transaction.support.TransactionSynchronization;
|
||||||
@@ -21,16 +23,10 @@ import org.springframework.transaction.support.TransactionSynchronizationManager
|
|||||||
import org.springframework.web.multipart.MultipartFile;
|
import org.springframework.web.multipart.MultipartFile;
|
||||||
|
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.nio.file.Files;
|
|
||||||
import java.nio.file.Path;
|
|
||||||
import java.security.MessageDigest;
|
import java.security.MessageDigest;
|
||||||
import java.security.NoSuchAlgorithmException;
|
import java.security.NoSuchAlgorithmException;
|
||||||
import java.sql.PreparedStatement;
|
|
||||||
import java.sql.Statement;
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.HexFormat;
|
import java.util.HexFormat;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -38,12 +34,15 @@ import java.util.Locale;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
|
import java.util.function.LongSupplier;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
public class PersonalIngestionService {
|
public class PersonalIngestionService {
|
||||||
|
|
||||||
private static final String ITEM_NOT_FOUND = "PERSONAL_ITEM_NOT_FOUND";
|
private static final String ITEM_NOT_FOUND = "PERSONAL_ITEM_NOT_FOUND";
|
||||||
|
private static final Pattern SAFE_TENANT = Pattern.compile("[A-Za-z0-9_-]{1,20}");
|
||||||
private static final Set<String> SUPPORTED_FILE_SUFFIXES = Set.of(
|
private static final Set<String> SUPPORTED_FILE_SUFFIXES = Set.of(
|
||||||
"txt", "md", "markdown", "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx"
|
"txt", "md", "markdown", "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx"
|
||||||
);
|
);
|
||||||
@@ -53,55 +52,54 @@ public class PersonalIngestionService {
|
|||||||
private final PersonalKnowledgeProperties properties;
|
private final PersonalKnowledgeProperties properties;
|
||||||
private final ISysOssService ossService;
|
private final ISysOssService ossService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
private final PersonalObjectStore objectStore;
|
||||||
|
private final LongSupplier itemIdSupplier;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
public PersonalIngestionService(JdbcTemplate jdbcTemplate, PersonalSpaceService spaceService,
|
public PersonalIngestionService(JdbcTemplate jdbcTemplate, PersonalSpaceService spaceService,
|
||||||
PersonalKnowledgeProperties properties, ISysOssService ossService,
|
PersonalKnowledgeProperties properties, ISysOssService ossService,
|
||||||
ObjectMapper objectMapper) {
|
ObjectMapper objectMapper) {
|
||||||
|
this(jdbcTemplate, spaceService, properties, ossService, objectMapper,
|
||||||
|
new DefaultPersonalObjectStore(jdbcTemplate), IdWorker::getId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private PersonalIngestionService(JdbcTemplate jdbcTemplate, PersonalSpaceService spaceService,
|
||||||
|
PersonalKnowledgeProperties properties, ISysOssService ossService,
|
||||||
|
ObjectMapper objectMapper, PersonalObjectStore objectStore,
|
||||||
|
LongSupplier itemIdSupplier) {
|
||||||
this.jdbcTemplate = jdbcTemplate;
|
this.jdbcTemplate = jdbcTemplate;
|
||||||
this.spaceService = spaceService;
|
this.spaceService = spaceService;
|
||||||
this.properties = properties;
|
this.properties = properties;
|
||||||
this.ossService = ossService;
|
this.ossService = ossService;
|
||||||
this.objectMapper = objectMapper;
|
this.objectMapper = objectMapper;
|
||||||
|
this.objectStore = objectStore;
|
||||||
|
this.itemIdSupplier = itemIdSupplier;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static PersonalIngestionService forTest(JdbcTemplate jdbcTemplate, PersonalSpaceService spaceService,
|
||||||
|
PersonalKnowledgeProperties properties, ISysOssService ossService,
|
||||||
|
ObjectMapper objectMapper, PersonalObjectStore objectStore,
|
||||||
|
LongSupplier itemIdSupplier) {
|
||||||
|
return new PersonalIngestionService(jdbcTemplate, spaceService, properties, ossService, objectMapper,
|
||||||
|
objectStore, itemIdSupplier);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public ItemCreatedResponse createText(PersonalOwner owner, TextItemRequest request) {
|
public ItemCreatedResponse createText(PersonalOwner owner, TextItemRequest request) {
|
||||||
|
validateOwner(owner);
|
||||||
if (request == null || request.content() == null || request.content().isBlank()) {
|
if (request == null || request.content() == null || request.content().isBlank()) {
|
||||||
throw new ServiceException("PERSONAL_TEXT_EMPTY");
|
throw new ServiceException("PERSONAL_TEXT_EMPTY");
|
||||||
}
|
}
|
||||||
byte[] bytes = request.content().getBytes(StandardCharsets.UTF_8);
|
byte[] bytes = request.content().getBytes(StandardCharsets.UTF_8);
|
||||||
validateSize(bytes.length);
|
validateSize(bytes.length);
|
||||||
String hash = sha256(bytes);
|
return create(owner, "TEXT", cleanTitle(request.title(), "文字资料"), "txt", "text/plain", bytes,
|
||||||
ItemCreatedResponse duplicate = duplicate(owner, hash);
|
request.capturedAt(), request.tags());
|
||||||
if (duplicate != null) {
|
|
||||||
return duplicate;
|
|
||||||
}
|
|
||||||
|
|
||||||
long spaceId = spaceService.reserve(owner, bytes.length);
|
|
||||||
Path temporary = null;
|
|
||||||
SysOssVo uploaded;
|
|
||||||
try {
|
|
||||||
temporary = Files.createTempFile("personal-" + UUID.randomUUID(), ".txt");
|
|
||||||
Files.write(temporary, bytes);
|
|
||||||
uploaded = ossService.upload(temporary.toFile());
|
|
||||||
} catch (IOException ex) {
|
|
||||||
throw new ServiceException("PERSONAL_TEXT_STORAGE_FAILED");
|
|
||||||
} finally {
|
|
||||||
if (temporary != null) {
|
|
||||||
try {
|
|
||||||
Files.deleteIfExists(temporary);
|
|
||||||
} catch (IOException ex) {
|
|
||||||
log.warn("Unable to remove temporary personal text object");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return persist(owner, spaceId, "TEXT", cleanTitle(request.title(), "文字资料"),
|
|
||||||
"text/plain", bytes.length, hash, request.capturedAt(), request.tags(), uploaded);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Transactional
|
@Transactional
|
||||||
public ItemCreatedResponse createFile(PersonalOwner owner, MultipartFile file, String title,
|
public ItemCreatedResponse createFile(PersonalOwner owner, MultipartFile file, String title,
|
||||||
LocalDateTime capturedAt) {
|
LocalDateTime capturedAt) {
|
||||||
|
validateOwner(owner);
|
||||||
validateFile(file);
|
validateFile(file);
|
||||||
byte[] bytes;
|
byte[] bytes;
|
||||||
try {
|
try {
|
||||||
@@ -109,23 +107,14 @@ public class PersonalIngestionService {
|
|||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
throw new ServiceException("PERSONAL_FILE_READ_FAILED");
|
throw new ServiceException("PERSONAL_FILE_READ_FAILED");
|
||||||
}
|
}
|
||||||
String hash = sha256(bytes);
|
|
||||||
ItemCreatedResponse duplicate = duplicate(owner, hash);
|
|
||||||
if (duplicate != null) {
|
|
||||||
return duplicate;
|
|
||||||
}
|
|
||||||
|
|
||||||
validateSize(bytes.length);
|
validateSize(bytes.length);
|
||||||
long spaceId = spaceService.reserve(owner, bytes.length);
|
|
||||||
String safeUploadName = "personal-" + UUID.randomUUID() + suffixWithDot(file.getOriginalFilename());
|
|
||||||
SysOssVo uploaded = ossService.upload(new SafeMultipartFile(
|
|
||||||
safeUploadName, cleanMime(file.getContentType()), bytes));
|
|
||||||
String originalName = safeFileName(file.getOriginalFilename());
|
String originalName = safeFileName(file.getOriginalFilename());
|
||||||
return persist(owner, spaceId, "FILE", cleanTitle(title, originalName),
|
return create(owner, "FILE", cleanTitle(title, originalName), suffix(originalName),
|
||||||
cleanMime(file.getContentType()), bytes.length, hash, capturedAt, List.of(), uploaded);
|
cleanMime(file.getContentType()), bytes, capturedAt, List.of());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void retry(PersonalOwner owner, long itemId) {
|
public void retry(PersonalOwner owner, long itemId) {
|
||||||
|
validateOwner(owner);
|
||||||
int updated = jdbcTemplate.update("""
|
int updated = jdbcTemplate.update("""
|
||||||
update aihr_personal_item
|
update aihr_personal_item
|
||||||
set status = 'QUEUED', error_code = null, error_message = null,
|
set status = 'QUEUED', error_code = null, error_message = null,
|
||||||
@@ -138,37 +127,50 @@ public class PersonalIngestionService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private ItemCreatedResponse persist(PersonalOwner owner, long spaceId, String sourceType, String title,
|
private ItemCreatedResponse create(PersonalOwner owner, String sourceType, String title, String suffix,
|
||||||
String mimeType, long size, String hash, LocalDateTime capturedAt,
|
String mimeType, byte[] bytes, LocalDateTime capturedAt, List<String> tags) {
|
||||||
List<String> tags, SysOssVo uploaded) {
|
String hash = sha256(bytes);
|
||||||
|
// reserve locks the current owner's space row. Dedupe must happen while that lock is held.
|
||||||
|
long spaceId = spaceService.reserve(owner, bytes.length);
|
||||||
|
ItemCreatedResponse duplicate = duplicate(owner, spaceId, hash);
|
||||||
|
if (duplicate != null) {
|
||||||
|
return duplicate;
|
||||||
|
}
|
||||||
|
|
||||||
|
long itemId = positiveId(itemIdSupplier.getAsLong());
|
||||||
|
String objectKey = objectKey(owner, itemId, suffix);
|
||||||
|
SysOssVo uploaded = objectStore.upload(owner, itemId, objectKey, suffix, mimeType, bytes);
|
||||||
|
return persist(owner, spaceId, itemId, sourceType, title, mimeType, bytes.length, hash, capturedAt,
|
||||||
|
tags, objectKey, uploaded);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ItemCreatedResponse persist(PersonalOwner owner, long spaceId, long itemId, String sourceType,
|
||||||
|
String title, String mimeType, long size, String hash,
|
||||||
|
LocalDateTime capturedAt, List<String> tags, String objectKey,
|
||||||
|
SysOssVo uploaded) {
|
||||||
if (uploaded == null || uploaded.getOssId() == null) {
|
if (uploaded == null || uploaded.getOssId() == null) {
|
||||||
throw new ServiceException("PERSONAL_OSS_UPLOAD_FAILED");
|
throw new ServiceException("PERSONAL_OSS_UPLOAD_FAILED");
|
||||||
}
|
}
|
||||||
boolean deferredCleanup = registerRollbackCleanup(uploaded);
|
boolean deferredCleanup = registerRollbackCleanup(uploaded);
|
||||||
try {
|
try {
|
||||||
KeyHolder keyHolder = new GeneratedKeyHolder();
|
int bound = jdbcTemplate.update("""
|
||||||
jdbcTemplate.update(connection -> {
|
update sys_oss
|
||||||
PreparedStatement statement = connection.prepareStatement("""
|
set ext1 = ?, update_time = now(), update_by = ?
|
||||||
insert into aihr_personal_item
|
where tenant_id = ? and oss_id = ? and create_by = ? and file_name = ?
|
||||||
(tenant_id, space_id, owner_user_id, source_type, title, oss_id, mime_type,
|
""", personalOssExt(itemId), owner.userId(), owner.tenantId(), uploaded.getOssId(),
|
||||||
size_bytes, content_hash, status, tags_json, captured_at, create_time, update_time)
|
owner.userId(), objectKey);
|
||||||
values (?, ?, ?, ?, ?, ?, ?, ?, ?, 'QUEUED', ?, ?, now(), now())
|
if (bound != 1) {
|
||||||
""", Statement.RETURN_GENERATED_KEYS);
|
throw new ServiceException("PERSONAL_OSS_BIND_FAILED");
|
||||||
statement.setString(1, owner.tenantId());
|
}
|
||||||
statement.setLong(2, spaceId);
|
int inserted = jdbcTemplate.update("""
|
||||||
statement.setLong(3, owner.userId());
|
insert into aihr_personal_item
|
||||||
statement.setString(4, sourceType);
|
(id, tenant_id, space_id, owner_user_id, source_type, title, oss_id, mime_type,
|
||||||
statement.setString(5, title);
|
size_bytes, content_hash, status, tags_json, captured_at, create_time, update_time)
|
||||||
statement.setLong(6, uploaded.getOssId());
|
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'QUEUED', ?, ?, now(), now())
|
||||||
statement.setString(7, mimeType);
|
""", itemId, owner.tenantId(), spaceId, owner.userId(), sourceType, title,
|
||||||
statement.setLong(8, size);
|
uploaded.getOssId(), mimeType, size, hash, tagsJson(tags),
|
||||||
statement.setString(9, hash);
|
capturedAt == null ? LocalDateTime.now() : capturedAt);
|
||||||
statement.setString(10, tagsJson(tags));
|
if (inserted != 1) {
|
||||||
statement.setObject(11, capturedAt == null ? LocalDateTime.now() : capturedAt);
|
|
||||||
return statement;
|
|
||||||
}, keyHolder);
|
|
||||||
Number key = keyHolder.getKey();
|
|
||||||
if (key == null) {
|
|
||||||
throw new ServiceException("PERSONAL_ITEM_CREATE_FAILED");
|
throw new ServiceException("PERSONAL_ITEM_CREATE_FAILED");
|
||||||
}
|
}
|
||||||
int counterUpdated = jdbcTemplate.update("""
|
int counterUpdated = jdbcTemplate.update("""
|
||||||
@@ -179,24 +181,24 @@ public class PersonalIngestionService {
|
|||||||
if (counterUpdated != 1) {
|
if (counterUpdated != 1) {
|
||||||
throw new ServiceException("PERSONAL_SPACE_NOT_AVAILABLE");
|
throw new ServiceException("PERSONAL_SPACE_NOT_AVAILABLE");
|
||||||
}
|
}
|
||||||
return new ItemCreatedResponse(key.longValue(), "QUEUED", null);
|
return new ItemCreatedResponse(itemId, "QUEUED", null);
|
||||||
} catch (RuntimeException ex) {
|
} catch (RuntimeException ex) {
|
||||||
if (!deferredCleanup) {
|
if (!deferredCleanup) {
|
||||||
cleanupOss(uploaded.getOssId());
|
cleanupOss(uploaded);
|
||||||
}
|
}
|
||||||
throw ex;
|
throw ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private ItemCreatedResponse duplicate(PersonalOwner owner, String hash) {
|
private ItemCreatedResponse duplicate(PersonalOwner owner, long spaceId, String hash) {
|
||||||
List<Map<String, Object>> rows = jdbcTemplate.queryForList("""
|
List<Map<String, Object>> rows = jdbcTemplate.queryForList("""
|
||||||
select id, status
|
select id, status
|
||||||
from aihr_personal_item
|
from aihr_personal_item
|
||||||
where tenant_id = ? and owner_user_id = ? and content_hash = ?
|
where tenant_id = ? and owner_user_id = ? and space_id = ? and content_hash = ?
|
||||||
and status <> 'DELETED'
|
and status <> 'DELETED'
|
||||||
order by id desc
|
order by id desc
|
||||||
limit 1
|
limit 1
|
||||||
""", owner.tenantId(), owner.userId(), hash);
|
""", owner.tenantId(), owner.userId(), spaceId, hash);
|
||||||
if (rows.isEmpty()) {
|
if (rows.isEmpty()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -210,8 +212,7 @@ public class PersonalIngestionService {
|
|||||||
throw new ServiceException("PERSONAL_FILE_EMPTY");
|
throw new ServiceException("PERSONAL_FILE_EMPTY");
|
||||||
}
|
}
|
||||||
validateSize(file.getSize());
|
validateSize(file.getSize());
|
||||||
String suffix = suffix(file.getOriginalFilename());
|
if (!SUPPORTED_FILE_SUFFIXES.contains(suffix(file.getOriginalFilename()))) {
|
||||||
if (!SUPPORTED_FILE_SUFFIXES.contains(suffix)) {
|
|
||||||
throw new ServiceException("PERSONAL_FILE_UNSUPPORTED");
|
throw new ServiceException("PERSONAL_FILE_UNSUPPORTED");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -236,32 +237,31 @@ public class PersonalIngestionService {
|
|||||||
@Override
|
@Override
|
||||||
public void afterCompletion(int status) {
|
public void afterCompletion(int status) {
|
||||||
if (status != STATUS_COMMITTED) {
|
if (status != STATUS_COMMITTED) {
|
||||||
cleanupRolledBackOss(uploaded);
|
cleanupOss(uploaded);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void cleanupRolledBackOss(SysOssVo uploaded) {
|
private void cleanupOss(SysOssVo uploaded) {
|
||||||
if (uploaded.getService() != null && !uploaded.getService().isBlank()
|
try {
|
||||||
&& uploaded.getFileName() != null && !uploaded.getFileName().isBlank()) {
|
objectStore.deletePhysical(uploaded);
|
||||||
try {
|
} catch (RuntimeException cleanupError) {
|
||||||
// sys_oss participates in the outer transaction and may already be rolled back here,
|
log.warn("Unable to clean personal OSS object id={}", uploaded.getOssId());
|
||||||
// so remove the physical object by its private object key before metadata cleanup.
|
}
|
||||||
OssFactory.instance(uploaded.getService()).delete(uploaded.getFileName());
|
try {
|
||||||
} catch (RuntimeException cleanupError) {
|
ossService.deleteWithValidByIds(List.of(uploaded.getOssId()), false);
|
||||||
log.warn("Unable to clean rolled-back personal OSS object id={}", uploaded.getOssId());
|
} catch (RuntimeException cleanupError) {
|
||||||
}
|
log.warn("Unable to clean personal OSS metadata id={}", uploaded.getOssId());
|
||||||
}
|
}
|
||||||
cleanupOss(uploaded.getOssId());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void cleanupOss(long ossId) {
|
private String personalOssExt(long itemId) {
|
||||||
try {
|
try {
|
||||||
ossService.deleteWithValidByIds(List.of(ossId), false);
|
return objectMapper.writeValueAsString(Map.of("source", "personal", "itemId", itemId));
|
||||||
} catch (RuntimeException cleanupError) {
|
} catch (JsonProcessingException ex) {
|
||||||
log.warn("Unable to clean personal OSS object id={}", ossId);
|
throw new ServiceException("PERSONAL_OSS_BIND_FAILED");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -280,6 +280,32 @@ public class PersonalIngestionService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static String objectKey(PersonalOwner owner, long itemId, String suffix) {
|
||||||
|
validateOwner(owner);
|
||||||
|
long safeItemId = positiveId(itemId);
|
||||||
|
String safeSuffix = suffix == null ? "" : suffix.toLowerCase(Locale.ROOT);
|
||||||
|
if (!SUPPORTED_FILE_SUFFIXES.contains(safeSuffix)) {
|
||||||
|
throw new ServiceException("PERSONAL_FILE_UNSUPPORTED");
|
||||||
|
}
|
||||||
|
String randomName = UUID.randomUUID().toString().replace("-", "");
|
||||||
|
return "personal/" + owner.tenantId() + "/" + owner.userId() + "/" + safeItemId + "/"
|
||||||
|
+ randomName + "." + safeSuffix;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void validateOwner(PersonalOwner owner) {
|
||||||
|
if (owner == null || owner.userId() <= 0 || owner.tenantId() == null
|
||||||
|
|| !SAFE_TENANT.matcher(owner.tenantId()).matches()) {
|
||||||
|
throw new ServiceException("PERSONAL_OWNER_INVALID");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static long positiveId(long id) {
|
||||||
|
if (id <= 0) {
|
||||||
|
throw new ServiceException("PERSONAL_ID_INVALID");
|
||||||
|
}
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
private static String sha256(byte[] bytes) {
|
private static String sha256(byte[] bytes) {
|
||||||
try {
|
try {
|
||||||
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
|
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
|
||||||
@@ -313,11 +339,6 @@ public class PersonalIngestionService {
|
|||||||
return dot < 0 ? "" : safe.substring(dot + 1).toLowerCase(Locale.ROOT);
|
return dot < 0 ? "" : safe.substring(dot + 1).toLowerCase(Locale.ROOT);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String suffixWithDot(String fileName) {
|
|
||||||
String suffix = suffix(fileName);
|
|
||||||
return suffix.isBlank() ? "" : "." + suffix;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String cleanMime(String value) {
|
private static String cleanMime(String value) {
|
||||||
if (value == null || value.isBlank()) {
|
if (value == null || value.isBlank()) {
|
||||||
return "application/octet-stream";
|
return "application/octet-stream";
|
||||||
@@ -327,55 +348,62 @@ public class PersonalIngestionService {
|
|||||||
return separator < 0 ? mime : mime.substring(0, separator).trim();
|
return separator < 0 ? mime : mime.substring(0, separator).trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final class SafeMultipartFile implements MultipartFile {
|
public interface PersonalObjectStore {
|
||||||
private final String fileName;
|
SysOssVo upload(PersonalOwner owner, long itemId, String objectKey, String suffix, String mimeType,
|
||||||
private final String contentType;
|
byte[] bytes);
|
||||||
private final byte[] bytes;
|
|
||||||
|
|
||||||
private SafeMultipartFile(String fileName, String contentType, byte[] bytes) {
|
void deletePhysical(SysOssVo uploaded);
|
||||||
this.fileName = fileName;
|
}
|
||||||
this.contentType = contentType;
|
|
||||||
this.bytes = bytes.clone();
|
private static final class DefaultPersonalObjectStore implements PersonalObjectStore {
|
||||||
|
private final JdbcTemplate jdbcTemplate;
|
||||||
|
|
||||||
|
private DefaultPersonalObjectStore(JdbcTemplate jdbcTemplate) {
|
||||||
|
this.jdbcTemplate = jdbcTemplate;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getName() {
|
public SysOssVo upload(PersonalOwner owner, long itemId, String objectKey, String suffix, String mimeType,
|
||||||
return "file";
|
byte[] bytes) {
|
||||||
|
OssClient storage = OssFactory.instance();
|
||||||
|
UploadResult result = storage.upload(
|
||||||
|
new ByteArrayInputStream(bytes), objectKey, (long) bytes.length, mimeType);
|
||||||
|
long ossId = IdWorker.getId();
|
||||||
|
String safeName = objectKey.substring(objectKey.lastIndexOf('/') + 1);
|
||||||
|
try {
|
||||||
|
int inserted = jdbcTemplate.update("""
|
||||||
|
insert into sys_oss
|
||||||
|
(oss_id, tenant_id, file_name, original_name, file_suffix, url, ext1,
|
||||||
|
create_time, create_by, update_time, update_by, service)
|
||||||
|
values (?, ?, ?, ?, ?, ?, null, now(), ?, now(), ?, ?)
|
||||||
|
""", ossId, owner.tenantId(), result.getFilename(), safeName, "." + suffix,
|
||||||
|
result.getUrl(), owner.userId(), owner.userId(), storage.getConfigKey());
|
||||||
|
if (inserted != 1) {
|
||||||
|
throw new ServiceException("PERSONAL_OSS_METADATA_FAILED");
|
||||||
|
}
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
try {
|
||||||
|
storage.delete(result.getFilename());
|
||||||
|
} catch (RuntimeException cleanupError) {
|
||||||
|
log.warn("Unable to clean personal object after metadata failure");
|
||||||
|
}
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
SysOssVo uploaded = new SysOssVo();
|
||||||
|
uploaded.setOssId(ossId);
|
||||||
|
uploaded.setFileName(result.getFilename());
|
||||||
|
uploaded.setOriginalName(safeName);
|
||||||
|
uploaded.setFileSuffix("." + suffix);
|
||||||
|
uploaded.setUrl(result.getUrl());
|
||||||
|
uploaded.setService(storage.getConfigKey());
|
||||||
|
return uploaded;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getOriginalFilename() {
|
public void deletePhysical(SysOssVo uploaded) {
|
||||||
return fileName;
|
if (uploaded.getService() != null && uploaded.getFileName() != null) {
|
||||||
}
|
OssFactory.instance(uploaded.getService()).delete(uploaded.getFileName());
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public String getContentType() {
|
|
||||||
return contentType;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean isEmpty() {
|
|
||||||
return bytes.length == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public long getSize() {
|
|
||||||
return bytes.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public byte[] getBytes() {
|
|
||||||
return bytes.clone();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public InputStream getInputStream() {
|
|
||||||
return new ByteArrayInputStream(bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void transferTo(File destination) throws IOException {
|
|
||||||
Files.write(destination.toPath(), bytes);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+11
-4
@@ -90,7 +90,8 @@ public class PersonalIngestionWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
StoredObject stored = objectReader.read(item.ossId(), maxInputBytes);
|
StoredObject stored = objectReader.read(
|
||||||
|
item.ossId(), ownerObjectPrefix(item), item.ownerUserId(), maxInputBytes);
|
||||||
ParsedDocument document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes());
|
ParsedDocument document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes());
|
||||||
List<String> chunks = document.chunks(CHUNK_SIZE, CHUNK_OVERLAP);
|
List<String> chunks = document.chunks(CHUNK_SIZE, CHUNK_OVERLAP);
|
||||||
if (chunks.isEmpty()) {
|
if (chunks.isEmpty()) {
|
||||||
@@ -156,10 +157,12 @@ public class PersonalIngestionWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static StoredObjectReader defaultReader(ISysOssService ossService) {
|
private static StoredObjectReader defaultReader(ISysOssService ossService) {
|
||||||
return (ossId, maxBytes) -> {
|
return (ossId, expectedPrefix, ownerUserId, maxBytes) -> {
|
||||||
SysOssVo object = ossService.getById(ossId);
|
SysOssVo object = ossService.getById(ossId);
|
||||||
if (object == null || object.getFileName() == null || object.getFileName().isBlank()
|
if (object == null || object.getFileName() == null || object.getFileName().isBlank()
|
||||||
|| object.getService() == null || object.getService().isBlank()) {
|
|| object.getService() == null || object.getService().isBlank()
|
||||||
|
|| object.getCreateBy() == null || object.getCreateBy() != ownerUserId
|
||||||
|
|| !object.getFileName().startsWith(expectedPrefix)) {
|
||||||
throw new IOException("personal source object is unavailable");
|
throw new IOException("personal source object is unavailable");
|
||||||
}
|
}
|
||||||
OssClient storage = OssFactory.instance(object.getService());
|
OssClient storage = OssFactory.instance(object.getService());
|
||||||
@@ -246,9 +249,13 @@ public class PersonalIngestionWorker {
|
|||||||
return safe.isBlank() ? "personal-object" : safe;
|
return safe.isBlank() ? "personal-object" : safe;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static String ownerObjectPrefix(Item item) {
|
||||||
|
return "personal/" + item.tenantId() + "/" + item.ownerUserId() + "/" + item.id() + "/";
|
||||||
|
}
|
||||||
|
|
||||||
@FunctionalInterface
|
@FunctionalInterface
|
||||||
public interface StoredObjectReader {
|
public interface StoredObjectReader {
|
||||||
StoredObject read(long ossId, long maxBytes) throws Exception;
|
StoredObject read(long ossId, String expectedPrefix, long ownerUserId, long maxBytes) throws Exception;
|
||||||
}
|
}
|
||||||
|
|
||||||
public record StoredObject(String fileName, byte[] bytes) {
|
public record StoredObject(String fileName, byte[] bytes) {
|
||||||
|
|||||||
+172
-150
@@ -1,9 +1,11 @@
|
|||||||
package org.dromara.aihr.personal;
|
package org.dromara.aihr.personal;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import org.dromara.aihr.personal.domain.PersonalAssistantDto.ItemCreatedResponse;
|
import org.dromara.aihr.personal.domain.PersonalAssistantDto.ItemCreatedResponse;
|
||||||
import org.dromara.aihr.personal.domain.PersonalAssistantDto.TextItemRequest;
|
import org.dromara.aihr.personal.domain.PersonalAssistantDto.TextItemRequest;
|
||||||
import org.dromara.aihr.personal.service.PersonalIngestionService;
|
import org.dromara.aihr.personal.service.PersonalIngestionService;
|
||||||
|
import org.dromara.aihr.personal.service.PersonalIngestionService.PersonalObjectStore;
|
||||||
import org.dromara.aihr.personal.service.PersonalSpaceService;
|
import org.dromara.aihr.personal.service.PersonalSpaceService;
|
||||||
import org.dromara.aihr.personal.support.PersonalKnowledgeProperties;
|
import org.dromara.aihr.personal.support.PersonalKnowledgeProperties;
|
||||||
import org.dromara.aihr.personal.support.PersonalOwner;
|
import org.dromara.aihr.personal.support.PersonalOwner;
|
||||||
@@ -16,10 +18,7 @@ import org.mockito.ArgumentCaptor;
|
|||||||
import org.mockito.InOrder;
|
import org.mockito.InOrder;
|
||||||
import org.springframework.aop.framework.ProxyFactory;
|
import org.springframework.aop.framework.ProxyFactory;
|
||||||
import org.springframework.jdbc.core.JdbcTemplate;
|
import org.springframework.jdbc.core.JdbcTemplate;
|
||||||
import org.springframework.jdbc.core.PreparedStatementCreator;
|
|
||||||
import org.springframework.jdbc.support.KeyHolder;
|
|
||||||
import org.springframework.mock.web.MockMultipartFile;
|
import org.springframework.mock.web.MockMultipartFile;
|
||||||
import org.springframework.transaction.annotation.Transactional;
|
|
||||||
import org.springframework.transaction.TransactionDefinition;
|
import org.springframework.transaction.TransactionDefinition;
|
||||||
import org.springframework.transaction.annotation.AnnotationTransactionAttributeSource;
|
import org.springframework.transaction.annotation.AnnotationTransactionAttributeSource;
|
||||||
import org.springframework.transaction.interceptor.TransactionInterceptor;
|
import org.springframework.transaction.interceptor.TransactionInterceptor;
|
||||||
@@ -32,17 +31,20 @@ import java.time.LocalDateTime;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyLong;
|
||||||
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
import static org.mockito.ArgumentMatchers.contains;
|
import static org.mockito.ArgumentMatchers.contains;
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
import static org.mockito.Mockito.inOrder;
|
import static org.mockito.Mockito.inOrder;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.never;
|
import static org.mockito.Mockito.never;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
import static org.mockito.Mockito.verify;
|
import static org.mockito.Mockito.verify;
|
||||||
import static org.mockito.Mockito.verifyNoInteractions;
|
import static org.mockito.Mockito.verifyNoInteractions;
|
||||||
import static org.mockito.Mockito.when;
|
import static org.mockito.Mockito.when;
|
||||||
@@ -54,224 +56,240 @@ class PersonalIngestionServiceTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
void failedItemRetriesThroughOwnerScopedQueuedState() {
|
void failedItemRetriesThroughOwnerScopedQueuedState() {
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
Fixture fixture = fixture(100L);
|
||||||
when(jdbc.update(contains("status = 'QUEUED'"), eq("000000"), eq(101L), eq(9L)))
|
when(fixture.jdbc.update(contains("status = 'QUEUED'"), eq("000000"), eq(101L), eq(9L)))
|
||||||
.thenReturn(1);
|
.thenReturn(1);
|
||||||
PersonalIngestionService service = service(jdbc, mock(PersonalSpaceService.class), mock(ISysOssService.class));
|
|
||||||
|
|
||||||
service.retry(OWNER, 9L);
|
fixture.service.retry(OWNER, 9L);
|
||||||
|
|
||||||
verify(jdbc).update(contains("status = 'QUEUED'"), eq("000000"), eq(101L), eq(9L));
|
verify(fixture.jdbc).update(contains("status = 'QUEUED'"), eq("000000"), eq(101L), eq(9L));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void retryDoesNotDiscloseMissingOrForeignItem() {
|
void retryDoesNotDiscloseMissingOrForeignItem() {
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
Fixture fixture = fixture(100L);
|
||||||
PersonalIngestionService service = service(jdbc, mock(PersonalSpaceService.class), mock(ISysOssService.class));
|
|
||||||
|
|
||||||
ServiceException error = assertThrows(ServiceException.class, () -> service.retry(OWNER, 9L));
|
ServiceException error = assertThrows(ServiceException.class, () -> fixture.service.retry(OWNER, 9L));
|
||||||
|
|
||||||
assertEquals("PERSONAL_ITEM_NOT_FOUND", error.getMessage());
|
assertEquals("PERSONAL_ITEM_NOT_FOUND", error.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void fileLargerThanConfiguredLimitIsRejectedBeforeOssUpload() {
|
void invalidFilesAreRejectedBeforeObjectStorage() {
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
Fixture fixture = fixture(100L);
|
||||||
ISysOssService oss = mock(ISysOssService.class);
|
MockMultipartFile large = new MockMultipartFile(
|
||||||
MockMultipartFile file = new MockMultipartFile(
|
|
||||||
"file", "large.pdf", "application/pdf", new byte[21 * 1024 * 1024]);
|
"file", "large.pdf", "application/pdf", new byte[21 * 1024 * 1024]);
|
||||||
PersonalIngestionService service = service(jdbc, mock(PersonalSpaceService.class), oss);
|
|
||||||
|
|
||||||
ServiceException error = assertThrows(ServiceException.class,
|
assertEquals("PERSONAL_FILE_TOO_LARGE", assertThrows(ServiceException.class,
|
||||||
() -> service.createFile(OWNER, file, null, null));
|
() -> fixture.service.createFile(OWNER, large, null, null)).getMessage());
|
||||||
|
assertThrows(ServiceException.class, () -> fixture.service.createFile(
|
||||||
assertEquals("PERSONAL_FILE_TOO_LARGE", error.getMessage());
|
OWNER, new MockMultipartFile("file", "script.exe", "application/octet-stream", new byte[]{1}),
|
||||||
verifyNoInteractions(oss);
|
null, null));
|
||||||
verifyNoInteractions(jdbc);
|
assertThrows(ServiceException.class, () -> fixture.service.createFile(
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void unsupportedAndEmptyFilesAreRejectedBeforeOssUpload() {
|
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
|
||||||
ISysOssService oss = mock(ISysOssService.class);
|
|
||||||
PersonalIngestionService service = service(jdbc, mock(PersonalSpaceService.class), oss);
|
|
||||||
|
|
||||||
assertThrows(ServiceException.class, () -> service.createFile(
|
|
||||||
OWNER, new MockMultipartFile("file", "script.exe", "application/octet-stream", new byte[]{1}), null, null));
|
|
||||||
assertThrows(ServiceException.class, () -> service.createFile(
|
|
||||||
OWNER, new MockMultipartFile("file", "empty.txt", "text/plain", new byte[0]), null, null));
|
OWNER, new MockMultipartFile("file", "empty.txt", "text/plain", new byte[0]), null, null));
|
||||||
|
|
||||||
verifyNoInteractions(oss);
|
verifyNoInteractions(fixture.store, fixture.oss, fixture.jdbc, fixture.spaces);
|
||||||
verifyNoInteractions(jdbc);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void createTextReservesUploadsInsertsAndMutatesCountersOnce() {
|
void textCreationLocksBeforeDedupeAndUsesIsolatedObjectKeyAndExtBinding() throws Exception {
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
Fixture fixture = fixture(100L);
|
||||||
PersonalSpaceService spaces = mock(PersonalSpaceService.class);
|
stubSuccessfulCreate(fixture, OWNER, 7L, 18L, 81L);
|
||||||
ISysOssService oss = mock(ISysOssService.class);
|
|
||||||
when(spaces.reserve(eq(OWNER), eq(18L))).thenReturn(7L);
|
|
||||||
when(jdbc.queryForList(contains("content_hash"), eq("000000"), eq(101L), any(String.class)))
|
|
||||||
.thenReturn(List.of());
|
|
||||||
when(oss.upload(any(java.io.File.class))).thenReturn(oss(81L));
|
|
||||||
generatedId(jdbc, 99L);
|
|
||||||
when(jdbc.update(contains("used_bytes = used_bytes +"), eq(18L), eq("000000"), eq(101L), eq(7L)))
|
|
||||||
.thenReturn(1);
|
|
||||||
PersonalIngestionService service = service(jdbc, spaces, oss);
|
|
||||||
|
|
||||||
ItemCreatedResponse response = service.createText(
|
ItemCreatedResponse response = fixture.service.createText(OWNER,
|
||||||
OWNER, new TextItemRequest("周报", "保洁巡检记录", LocalDateTime.of(2026, 7, 12, 9, 0), List.of("保洁")));
|
new TextItemRequest("周报", "保洁巡检记录", LocalDateTime.of(2026, 7, 12, 9, 0), List.of("保洁")));
|
||||||
|
|
||||||
assertEquals(99L, response.itemId());
|
|
||||||
assertEquals("QUEUED", response.status());
|
|
||||||
assertEquals(null, response.duplicateOf());
|
|
||||||
InOrder order = inOrder(spaces, oss, jdbc);
|
|
||||||
order.verify(spaces).reserve(OWNER, 18L);
|
|
||||||
order.verify(oss).upload(any(java.io.File.class));
|
|
||||||
order.verify(jdbc).update(any(PreparedStatementCreator.class), any(KeyHolder.class));
|
|
||||||
order.verify(jdbc).update(contains("used_bytes = used_bytes +"), eq(18L), eq("000000"), eq(101L), eq(7L));
|
|
||||||
assertNotNull(PersonalIngestionService.class.getAnnotation(org.springframework.stereotype.Service.class));
|
|
||||||
assertNotNull(method("createText", PersonalOwner.class, TextItemRequest.class).getAnnotation(Transactional.class));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void createFileDoesNotDeduplicateAcrossOwners() throws Exception {
|
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
|
||||||
PersonalSpaceService spaces = mock(PersonalSpaceService.class);
|
|
||||||
ISysOssService oss = mock(ISysOssService.class);
|
|
||||||
MockMultipartFile file = new MockMultipartFile(
|
|
||||||
"file", "notes.txt", "text/plain", "same".getBytes(StandardCharsets.UTF_8));
|
|
||||||
when(jdbc.queryForList(contains("content_hash"), eq("000000"), eq(101L), any(String.class)))
|
|
||||||
.thenReturn(List.of());
|
|
||||||
when(spaces.reserve(OWNER, 4L)).thenReturn(7L);
|
|
||||||
when(oss.upload(any(org.springframework.web.multipart.MultipartFile.class))).thenReturn(oss(82L));
|
|
||||||
generatedId(jdbc, 100L);
|
|
||||||
when(jdbc.update(contains("used_bytes = used_bytes +"), eq(4L), eq("000000"), eq(101L), eq(7L)))
|
|
||||||
.thenReturn(1);
|
|
||||||
PersonalIngestionService service = service(jdbc, spaces, oss);
|
|
||||||
|
|
||||||
ItemCreatedResponse response = service.createFile(OWNER, file, null, null);
|
|
||||||
|
|
||||||
assertEquals(100L, response.itemId());
|
assertEquals(100L, response.itemId());
|
||||||
ArgumentCaptor<org.springframework.web.multipart.MultipartFile> uploaded =
|
assertEquals("QUEUED", response.status());
|
||||||
ArgumentCaptor.forClass(org.springframework.web.multipart.MultipartFile.class);
|
InOrder lockOrder = inOrder(fixture.spaces, fixture.jdbc);
|
||||||
verify(oss).upload(uploaded.capture());
|
lockOrder.verify(fixture.spaces).reserve(OWNER, 18L);
|
||||||
assertTrue(uploaded.getValue().getOriginalFilename().matches("personal-[0-9a-f-]+\\.txt"));
|
lockOrder.verify(fixture.jdbc).queryForList(contains("content_hash"),
|
||||||
assertEquals("same", new String(uploaded.getValue().getBytes(), StandardCharsets.UTF_8));
|
eq("000000"), eq(101L), eq(7L), anyString());
|
||||||
verify(jdbc).queryForList(contains("content_hash"), eq("000000"), eq(101L), any(String.class));
|
|
||||||
verify(jdbc, never()).queryForList(contains("content_hash"), eq("000000"), eq(202L), any(String.class));
|
ArgumentCaptor<String> key = ArgumentCaptor.forClass(String.class);
|
||||||
|
verify(fixture.store).upload(eq(OWNER), eq(100L), key.capture(), eq("txt"), eq("text/plain"),
|
||||||
|
any(byte[].class));
|
||||||
|
assertTrue(key.getValue().matches("personal/000000/101/100/[0-9a-f]{32}\\.txt"));
|
||||||
|
|
||||||
|
ArgumentCaptor<String> ext = ArgumentCaptor.forClass(String.class);
|
||||||
|
verify(fixture.jdbc).update(contains("update sys_oss"), ext.capture(), eq(101L), eq("000000"),
|
||||||
|
eq(81L), eq(101L), eq(key.getValue()));
|
||||||
|
JsonNode extJson = new ObjectMapper().readTree(ext.getValue());
|
||||||
|
assertEquals("personal", extJson.path("source").asText());
|
||||||
|
assertEquals(100L, extJson.path("itemId").asLong());
|
||||||
|
verify(fixture.jdbc).update(contains("insert into aihr_personal_item"), eq(100L), eq("000000"),
|
||||||
|
eq(7L), eq(101L), eq("TEXT"), eq("周报"), eq(81L), eq("text/plain"), eq(18L),
|
||||||
|
anyString(), anyString(), any(LocalDateTime.class));
|
||||||
|
verify(fixture.jdbc).update(contains("used_bytes = used_bytes +"),
|
||||||
|
eq(18L), eq("000000"), eq(101L), eq(7L));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void duplicateForSameOwnerReturnsExistingItemWithoutReservationOrUpload() {
|
void sameOwnerDuplicateIsCheckedUnderLockAndDoesNotUploadOrIncrementCounters() {
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
Fixture fixture = fixture(100L);
|
||||||
PersonalSpaceService spaces = mock(PersonalSpaceService.class);
|
when(fixture.spaces.reserve(OWNER, 4L)).thenReturn(7L);
|
||||||
ISysOssService oss = mock(ISysOssService.class);
|
when(fixture.jdbc.queryForList(contains("content_hash"),
|
||||||
|
eq("000000"), eq(101L), eq(7L), anyString()))
|
||||||
|
.thenReturn(List.of(Map.of("id", 77L, "status", "READY")));
|
||||||
MockMultipartFile file = new MockMultipartFile(
|
MockMultipartFile file = new MockMultipartFile(
|
||||||
"file", "notes.txt", "text/plain", "same".getBytes(StandardCharsets.UTF_8));
|
"file", "notes.txt", "text/plain", "same".getBytes(StandardCharsets.UTF_8));
|
||||||
when(jdbc.queryForList(contains("content_hash"), eq("000000"), eq(101L), any(String.class)))
|
|
||||||
.thenReturn(List.of(Map.of("id", 77L, "status", "READY")));
|
|
||||||
PersonalIngestionService service = service(jdbc, spaces, oss);
|
|
||||||
|
|
||||||
ItemCreatedResponse response = service.createFile(OWNER, file, null, null);
|
ItemCreatedResponse response = fixture.service.createFile(OWNER, file, null, null);
|
||||||
|
|
||||||
assertEquals(77L, response.itemId());
|
assertEquals(77L, response.itemId());
|
||||||
assertEquals(77L, response.duplicateOf());
|
assertEquals(77L, response.duplicateOf());
|
||||||
assertEquals("READY", response.status());
|
InOrder order = inOrder(fixture.spaces, fixture.jdbc);
|
||||||
verifyNoInteractions(spaces, oss);
|
order.verify(fixture.spaces).reserve(OWNER, 4L);
|
||||||
|
order.verify(fixture.jdbc).queryForList(contains("content_hash"),
|
||||||
|
eq("000000"), eq(101L), eq(7L), anyString());
|
||||||
|
verifyNoInteractions(fixture.store, fixture.oss);
|
||||||
|
verify(fixture.jdbc, never()).update(contains("used_bytes = used_bytes +"),
|
||||||
|
any(), any(), any(), any());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void failedPersistenceRegistersBestEffortOssCleanup() {
|
void sameContentAcrossOwnersCreatesSeparateObjects() {
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
AtomicLong ids = new AtomicLong(100L);
|
||||||
PersonalSpaceService spaces = mock(PersonalSpaceService.class);
|
Fixture fixture = fixture(ids::getAndIncrement);
|
||||||
ISysOssService oss = mock(ISysOssService.class);
|
PersonalOwner other = new PersonalOwner("000000", 202L, "ext-202");
|
||||||
|
when(fixture.spaces.reserve(OWNER, 4L)).thenReturn(7L);
|
||||||
|
when(fixture.spaces.reserve(other, 4L)).thenReturn(8L);
|
||||||
|
when(fixture.jdbc.queryForList(contains("content_hash"), any(), any(), any(), any()))
|
||||||
|
.thenReturn(List.of());
|
||||||
|
when(fixture.store.upload(any(), anyLong(), anyString(), eq("txt"), eq("text/plain"),
|
||||||
|
any(byte[].class))).thenAnswer(invocation -> oss(80L + invocation.<Long>getArgument(1),
|
||||||
|
invocation.getArgument(2)));
|
||||||
|
stubPersistence(fixture.jdbc);
|
||||||
MockMultipartFile file = new MockMultipartFile(
|
MockMultipartFile file = new MockMultipartFile(
|
||||||
"file", "notes.txt", "text/plain", "same".getBytes(StandardCharsets.UTF_8));
|
"file", "notes.txt", "text/plain", "same".getBytes(StandardCharsets.UTF_8));
|
||||||
when(jdbc.queryForList(contains("content_hash"), eq("000000"), eq(101L), any(String.class)))
|
|
||||||
.thenReturn(List.of());
|
ItemCreatedResponse first = fixture.service.createFile(OWNER, file, null, null);
|
||||||
when(spaces.reserve(OWNER, 4L)).thenReturn(7L);
|
ItemCreatedResponse second = fixture.service.createFile(other, file, null, null);
|
||||||
SysOssVo rolledBack = oss(82L);
|
|
||||||
rolledBack.setService(null);
|
assertEquals(100L, first.itemId());
|
||||||
when(oss.upload(any(org.springframework.web.multipart.MultipartFile.class))).thenReturn(rolledBack);
|
assertEquals(101L, second.itemId());
|
||||||
when(jdbc.update(any(PreparedStatementCreator.class), any(KeyHolder.class)))
|
verify(fixture.jdbc).queryForList(contains("content_hash"),
|
||||||
|
eq("000000"), eq(101L), eq(7L), anyString());
|
||||||
|
verify(fixture.jdbc).queryForList(contains("content_hash"),
|
||||||
|
eq("000000"), eq(202L), eq(8L), anyString());
|
||||||
|
verify(fixture.store, times(2)).upload(any(), anyLong(), anyString(), eq("txt"), eq("text/plain"),
|
||||||
|
any(byte[].class));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void failedItemPersistenceRollsBackAndCleansPhysicalObject() {
|
||||||
|
Fixture fixture = fixture(100L);
|
||||||
|
when(fixture.spaces.reserve(OWNER, 4L)).thenReturn(7L);
|
||||||
|
when(fixture.jdbc.queryForList(contains("content_hash"),
|
||||||
|
eq("000000"), eq(101L), eq(7L), anyString())).thenReturn(List.of());
|
||||||
|
SysOssVo uploaded = oss(81L, "personal/000000/101/100/a.txt");
|
||||||
|
when(fixture.store.upload(any(), anyLong(), anyString(), anyString(), anyString(), any(byte[].class)))
|
||||||
|
.thenReturn(uploaded);
|
||||||
|
when(fixture.jdbc.update(contains("update sys_oss"), any(), any(), any(), any(), any(), any()))
|
||||||
|
.thenReturn(1);
|
||||||
|
when(fixture.jdbc.update(contains("insert into aihr_personal_item"),
|
||||||
|
any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||||
.thenThrow(new IllegalStateException("db failed"));
|
.thenThrow(new IllegalStateException("db failed"));
|
||||||
TestTransactionManager transactions = new TestTransactionManager();
|
TestTransactionManager transactions = new TestTransactionManager();
|
||||||
PersonalIngestionService service = transactionalProxy(service(jdbc, spaces, oss), transactions);
|
PersonalIngestionService proxy = transactionalProxy(fixture.service, transactions);
|
||||||
|
MockMultipartFile file = new MockMultipartFile(
|
||||||
|
"file", "notes.txt", "text/plain", "same".getBytes(StandardCharsets.UTF_8));
|
||||||
|
|
||||||
assertThrows(IllegalStateException.class, () -> service.createFile(OWNER, file, null, null));
|
assertThrows(IllegalStateException.class, () -> proxy.createFile(OWNER, file, null, null));
|
||||||
|
|
||||||
verify(oss).deleteWithValidByIds(eq(List.of(82L)), eq(false));
|
|
||||||
assertEquals(1, transactions.rollbacks);
|
assertEquals(1, transactions.rollbacks);
|
||||||
|
verify(fixture.store).deletePhysical(uploaded);
|
||||||
|
verify(fixture.oss).deleteWithValidByIds(List.of(81L), false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void reservationAndCounterMutationRunInsideOneOuterTransaction() {
|
void reservationAndCountersShareOneOuterTransaction() {
|
||||||
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
Fixture fixture = fixture(100L);
|
||||||
PersonalSpaceService spaces = mock(PersonalSpaceService.class);
|
|
||||||
ISysOssService oss = mock(ISysOssService.class);
|
|
||||||
MockMultipartFile file = new MockMultipartFile(
|
|
||||||
"file", "notes.txt", "text/plain", "same".getBytes(StandardCharsets.UTF_8));
|
|
||||||
AtomicBoolean reserveInTransaction = new AtomicBoolean();
|
AtomicBoolean reserveInTransaction = new AtomicBoolean();
|
||||||
AtomicBoolean counterInTransaction = new AtomicBoolean();
|
AtomicBoolean counterInTransaction = new AtomicBoolean();
|
||||||
when(jdbc.queryForList(contains("content_hash"), eq("000000"), eq(101L), any(String.class)))
|
when(fixture.spaces.reserve(OWNER, 4L)).thenAnswer(invocation -> {
|
||||||
.thenReturn(List.of());
|
|
||||||
when(spaces.reserve(OWNER, 4L)).thenAnswer(invocation -> {
|
|
||||||
reserveInTransaction.set(TransactionSynchronizationManager.isActualTransactionActive());
|
reserveInTransaction.set(TransactionSynchronizationManager.isActualTransactionActive());
|
||||||
return 7L;
|
return 7L;
|
||||||
});
|
});
|
||||||
when(oss.upload(any(org.springframework.web.multipart.MultipartFile.class))).thenReturn(oss(82L));
|
when(fixture.jdbc.queryForList(contains("content_hash"), any(), any(), any(), any()))
|
||||||
generatedId(jdbc, 100L);
|
.thenReturn(List.of());
|
||||||
when(jdbc.update(contains("used_bytes = used_bytes +"), eq(4L), eq("000000"), eq(101L), eq(7L)))
|
when(fixture.store.upload(any(), anyLong(), anyString(), anyString(), anyString(), any(byte[].class)))
|
||||||
|
.thenReturn(oss(81L, "personal/000000/101/100/a.txt"));
|
||||||
|
when(fixture.jdbc.update(contains("update sys_oss"), any(), any(), any(), any(), any(), any()))
|
||||||
|
.thenReturn(1);
|
||||||
|
when(fixture.jdbc.update(contains("insert into aihr_personal_item"),
|
||||||
|
any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any()))
|
||||||
|
.thenReturn(1);
|
||||||
|
when(fixture.jdbc.update(contains("used_bytes = used_bytes +"), any(), any(), any(), any()))
|
||||||
.thenAnswer(invocation -> {
|
.thenAnswer(invocation -> {
|
||||||
counterInTransaction.set(TransactionSynchronizationManager.isActualTransactionActive());
|
counterInTransaction.set(TransactionSynchronizationManager.isActualTransactionActive());
|
||||||
return 1;
|
return 1;
|
||||||
});
|
});
|
||||||
TestTransactionManager transactions = new TestTransactionManager();
|
TestTransactionManager transactions = new TestTransactionManager();
|
||||||
PersonalIngestionService service = transactionalProxy(service(jdbc, spaces, oss), transactions);
|
PersonalIngestionService proxy = transactionalProxy(fixture.service, transactions);
|
||||||
|
MockMultipartFile file = new MockMultipartFile(
|
||||||
|
"file", "notes.txt", "text/plain", "same".getBytes(StandardCharsets.UTF_8));
|
||||||
|
|
||||||
service.createFile(OWNER, file, null, null);
|
proxy.createFile(OWNER, file, null, null);
|
||||||
|
|
||||||
assertTrue(reserveInTransaction.get());
|
assertTrue(reserveInTransaction.get());
|
||||||
assertTrue(counterInTransaction.get());
|
assertTrue(counterInTransaction.get());
|
||||||
assertEquals(1, transactions.commits);
|
assertEquals(1, transactions.commits);
|
||||||
assertEquals(0, transactions.rollbacks);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static PersonalIngestionService service(JdbcTemplate jdbc, PersonalSpaceService spaces, ISysOssService oss) {
|
@Test
|
||||||
return new PersonalIngestionService(jdbc, spaces, properties(), oss, new ObjectMapper());
|
void unsafeTenantIsRejectedBeforeStorage() {
|
||||||
|
Fixture fixture = fixture(100L);
|
||||||
|
PersonalOwner unsafe = new PersonalOwner("../000000", 101L, null);
|
||||||
|
|
||||||
|
assertEquals("PERSONAL_OWNER_INVALID", assertThrows(ServiceException.class,
|
||||||
|
() -> fixture.service.createText(unsafe, new TextItemRequest("x", "body", null, List.of())))
|
||||||
|
.getMessage());
|
||||||
|
|
||||||
|
verifyNoInteractions(fixture.spaces, fixture.store, fixture.jdbc, fixture.oss);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static PersonalKnowledgeProperties properties() {
|
private static Fixture fixture(long itemId) {
|
||||||
return new PersonalKnowledgeProperties();
|
return fixture(() -> itemId);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static SysOssVo oss(long id) {
|
private static Fixture fixture(java.util.function.LongSupplier itemIds) {
|
||||||
|
JdbcTemplate jdbc = mock(JdbcTemplate.class);
|
||||||
|
PersonalSpaceService spaces = mock(PersonalSpaceService.class);
|
||||||
|
ISysOssService ossService = mock(ISysOssService.class);
|
||||||
|
PersonalObjectStore store = mock(PersonalObjectStore.class);
|
||||||
|
PersonalIngestionService service = PersonalIngestionService.forTest(
|
||||||
|
jdbc, spaces, new PersonalKnowledgeProperties(), ossService, new ObjectMapper(), store, itemIds);
|
||||||
|
return new Fixture(jdbc, spaces, ossService, store, service);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void stubSuccessfulCreate(Fixture fixture, PersonalOwner owner, long spaceId, long bytes,
|
||||||
|
long ossId) {
|
||||||
|
when(fixture.spaces.reserve(owner, bytes)).thenReturn(spaceId);
|
||||||
|
when(fixture.jdbc.queryForList(contains("content_hash"),
|
||||||
|
eq(owner.tenantId()), eq(owner.userId()), eq(spaceId), anyString())).thenReturn(List.of());
|
||||||
|
when(fixture.store.upload(eq(owner), anyLong(), anyString(), anyString(), anyString(), any(byte[].class)))
|
||||||
|
.thenAnswer(invocation -> oss(ossId, invocation.getArgument(2)));
|
||||||
|
stubPersistence(fixture.jdbc);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void stubPersistence(JdbcTemplate jdbc) {
|
||||||
|
when(jdbc.update(contains("update sys_oss"), any(), any(), any(), any(), any(), any())).thenReturn(1);
|
||||||
|
when(jdbc.update(contains("insert into aihr_personal_item"),
|
||||||
|
any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(1);
|
||||||
|
when(jdbc.update(contains("used_bytes = used_bytes +"), any(), any(), any(), any())).thenReturn(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SysOssVo oss(long id, String fileName) {
|
||||||
SysOssVo result = new SysOssVo();
|
SysOssVo result = new SysOssVo();
|
||||||
result.setOssId(id);
|
result.setOssId(id);
|
||||||
result.setFileName("personal/random.txt");
|
result.setFileName(fileName);
|
||||||
result.setOriginalName("random.txt");
|
result.setOriginalName(fileName.substring(fileName.lastIndexOf('/') + 1));
|
||||||
result.setService("minio");
|
result.setService("minio");
|
||||||
result.setUrl("https://private.invalid/random.txt");
|
result.setUrl("https://private.invalid/" + fileName);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void generatedId(JdbcTemplate jdbc, long id) {
|
|
||||||
when(jdbc.update(any(PreparedStatementCreator.class), any(KeyHolder.class))).thenAnswer(invocation -> {
|
|
||||||
KeyHolder holder = invocation.getArgument(1);
|
|
||||||
holder.getKeyList().add(Map.of("GENERATED_KEY", id));
|
|
||||||
return 1;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private static java.lang.reflect.Method method(String name, Class<?>... types) {
|
|
||||||
try {
|
|
||||||
return PersonalIngestionService.class.getMethod(name, types);
|
|
||||||
} catch (NoSuchMethodException e) {
|
|
||||||
throw new AssertionError(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static PersonalIngestionService transactionalProxy(PersonalIngestionService target,
|
private static PersonalIngestionService transactionalProxy(PersonalIngestionService target,
|
||||||
TestTransactionManager transactionManager) {
|
TestTransactionManager transactionManager) {
|
||||||
ProxyFactory factory = new ProxyFactory(target);
|
ProxyFactory factory = new ProxyFactory(target);
|
||||||
@@ -284,6 +302,10 @@ class PersonalIngestionServiceTest {
|
|||||||
return (PersonalIngestionService) factory.getProxy();
|
return (PersonalIngestionService) factory.getProxy();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private record Fixture(JdbcTemplate jdbc, PersonalSpaceService spaces, ISysOssService oss,
|
||||||
|
PersonalObjectStore store, PersonalIngestionService service) {
|
||||||
|
}
|
||||||
|
|
||||||
private static final class TestTransactionManager extends AbstractPlatformTransactionManager {
|
private static final class TestTransactionManager extends AbstractPlatformTransactionManager {
|
||||||
private int commits;
|
private int commits;
|
||||||
private int rollbacks;
|
private int rollbacks;
|
||||||
|
|||||||
+10
-4
@@ -44,8 +44,12 @@ class PersonalIngestionWorkerTest {
|
|||||||
.thenReturn(1);
|
.thenReturn(1);
|
||||||
PersonalIngestionWorker worker = PersonalIngestionWorker.forTest(
|
PersonalIngestionWorker worker = PersonalIngestionWorker.forTest(
|
||||||
jdbc, mock(ISysOssService.class), parser, transactions,
|
jdbc, mock(ISysOssService.class), parser, transactions,
|
||||||
(ossId, maxBytes) -> new PersonalIngestionWorker.StoredObject(
|
(ossId, prefix, ownerUserId, maxBytes) -> {
|
||||||
"notes.txt", "same".getBytes(StandardCharsets.UTF_8)));
|
assertTrue(prefix.equals("personal/000000/101/9/"));
|
||||||
|
assertTrue(ownerUserId == 101L);
|
||||||
|
return new PersonalIngestionWorker.StoredObject(
|
||||||
|
"notes.txt", "same".getBytes(StandardCharsets.UTF_8));
|
||||||
|
});
|
||||||
|
|
||||||
assertTrue(worker.processNext());
|
assertTrue(worker.processNext());
|
||||||
|
|
||||||
@@ -61,7 +65,8 @@ class PersonalIngestionWorkerTest {
|
|||||||
when(jdbc.queryForList(contains("status = 'QUEUED'"))).thenReturn(List.of(item()));
|
when(jdbc.queryForList(contains("status = 'QUEUED'"))).thenReturn(List.of(item()));
|
||||||
PersonalIngestionWorker worker = PersonalIngestionWorker.forTest(
|
PersonalIngestionWorker worker = PersonalIngestionWorker.forTest(
|
||||||
jdbc, mock(ISysOssService.class), mock(KnowledgeDocumentParser.class), immediateTransactions(),
|
jdbc, mock(ISysOssService.class), mock(KnowledgeDocumentParser.class), immediateTransactions(),
|
||||||
(ossId, maxBytes) -> new PersonalIngestionWorker.StoredObject("notes.txt", new byte[]{1}));
|
(ossId, prefix, ownerUserId, maxBytes) ->
|
||||||
|
new PersonalIngestionWorker.StoredObject("notes.txt", new byte[]{1}));
|
||||||
|
|
||||||
assertFalse(worker.processNext());
|
assertFalse(worker.processNext());
|
||||||
|
|
||||||
@@ -80,7 +85,8 @@ class PersonalIngestionWorkerTest {
|
|||||||
KnowledgeDocumentParser.Failure.INVALID, "secret parser detail"));
|
KnowledgeDocumentParser.Failure.INVALID, "secret parser detail"));
|
||||||
PersonalIngestionWorker worker = PersonalIngestionWorker.forTest(
|
PersonalIngestionWorker worker = PersonalIngestionWorker.forTest(
|
||||||
jdbc, mock(ISysOssService.class), parser, immediateTransactions(),
|
jdbc, mock(ISysOssService.class), parser, immediateTransactions(),
|
||||||
(ossId, maxBytes) -> new PersonalIngestionWorker.StoredObject("notes.txt", new byte[]{1}));
|
(ossId, prefix, ownerUserId, maxBytes) ->
|
||||||
|
new PersonalIngestionWorker.StoredObject("notes.txt", new byte[]{1}));
|
||||||
|
|
||||||
assertTrue(worker.processNext());
|
assertTrue(worker.processNext());
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user