fix(personal): dedupe pending uploads
This commit is contained in:
+3
-1
@@ -447,7 +447,9 @@ public class PersonalIngestionService {
|
|||||||
and o.create_by = i.owner_user_id
|
and o.create_by = i.owner_user_id
|
||||||
where i.tenant_id = ? and i.owner_user_id = ? and i.space_id = ? and i.content_hash = ?
|
where i.tenant_id = ? and i.owner_user_id = ? and i.space_id = ? and i.content_hash = ?
|
||||||
and i.status <> 'DELETED'
|
and i.status <> 'DELETED'
|
||||||
and json_unquote(json_extract(o.ext1, '$.uploadState')) = 'READY'
|
and json_unquote(json_extract(o.ext1, '$.source')) = 'personal'
|
||||||
|
and cast(json_unquote(json_extract(o.ext1, '$.itemId')) as unsigned) = i.id
|
||||||
|
and json_unquote(json_extract(o.ext1, '$.uploadState')) in ('PENDING', 'READY')
|
||||||
order by i.id desc limit 1
|
order by i.id desc limit 1
|
||||||
""", owner.tenantId(), owner.userId(), spaceId, hash);
|
""", owner.tenantId(), owner.userId(), spaceId, hash);
|
||||||
if (rows.isEmpty()) return null;
|
if (rows.isEmpty()) return null;
|
||||||
|
|||||||
+50
-2
@@ -115,7 +115,7 @@ class PersonalIngestionServiceTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void duplicateIsResolvedOnlyFromReadyUploadUnderOwnerLock() {
|
void readyDuplicateIsResolvedUnderOwnerLockWithoutNewIntent() {
|
||||||
Fixture fixture = fixture();
|
Fixture fixture = fixture();
|
||||||
when(fixture.spaces.reserve(OWNER, 4L)).thenReturn(7L);
|
when(fixture.spaces.reserve(OWNER, 4L)).thenReturn(7L);
|
||||||
when(fixture.jdbc.queryForList(contains("content_hash"), eq("000000"), eq(101L), eq(7L), anyString()))
|
when(fixture.jdbc.queryForList(contains("content_hash"), eq("000000"), eq(101L), eq(7L), anyString()))
|
||||||
@@ -128,7 +128,7 @@ class PersonalIngestionServiceTest {
|
|||||||
assertEquals(77L, response.duplicateOf());
|
assertEquals(77L, response.duplicateOf());
|
||||||
ArgumentCaptor<String> dedupeSql = ArgumentCaptor.forClass(String.class);
|
ArgumentCaptor<String> dedupeSql = ArgumentCaptor.forClass(String.class);
|
||||||
verify(fixture.jdbc).queryForList(dedupeSql.capture(), eq("000000"), eq(101L), eq(7L), anyString());
|
verify(fixture.jdbc).queryForList(dedupeSql.capture(), eq("000000"), eq(101L), eq(7L), anyString());
|
||||||
assertTrue(dedupeSql.getValue().contains("$.uploadState')) = 'READY'"));
|
assertDedupeStatesAndBinding(dedupeSql.getValue());
|
||||||
verify(fixture.store).requirePrivateService();
|
verify(fixture.store).requirePrivateService();
|
||||||
verify(fixture.store, never()).uploadPhysical(anyString(), anyString(), anyString(), any(byte[].class));
|
verify(fixture.store, never()).uploadPhysical(anyString(), anyString(), anyString(), any(byte[].class));
|
||||||
verify(fixture.jdbc, never()).update(contains("insert into sys_oss"), any(), any(), any(), any(), any(),
|
verify(fixture.jdbc, never()).update(contains("insert into sys_oss"), any(), any(), any(), any(), any(),
|
||||||
@@ -136,6 +136,47 @@ class PersonalIngestionServiceTest {
|
|||||||
assertEquals(1, fixture.transactions.commits);
|
assertEquals(1, fixture.transactions.commits);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void pendingDuplicateIsResolvedWithoutNewObjectCounterOrUpload() {
|
||||||
|
Fixture fixture = fixture();
|
||||||
|
when(fixture.spaces.reserve(OWNER, 4L)).thenReturn(7L);
|
||||||
|
when(fixture.jdbc.queryForList(contains("content_hash"), eq("000000"), eq(101L), eq(7L), anyString()))
|
||||||
|
.thenReturn(List.of(Map.of("id", 77L, "status", "QUEUED")));
|
||||||
|
MockMultipartFile file = new MockMultipartFile("file", "notes.txt", "text/plain", new byte[]{1, 2, 3, 4});
|
||||||
|
|
||||||
|
ItemCreatedResponse response = fixture.service.createFile(OWNER, file, null, null);
|
||||||
|
|
||||||
|
assertEquals(77L, response.itemId());
|
||||||
|
assertEquals(77L, response.duplicateOf());
|
||||||
|
assertEquals("QUEUED", response.status());
|
||||||
|
verify(fixture.store, never()).uploadPhysical(anyString(), anyString(), anyString(), any(byte[].class));
|
||||||
|
verify(fixture.jdbc, never()).update(contains("insert into sys_oss"), any(), any(), any(), any(), any(),
|
||||||
|
any(), any(), any(), any());
|
||||||
|
verify(fixture.jdbc, never()).update(contains("insert into aihr_personal_item"), any(), any(), any(), any(),
|
||||||
|
any(), any(), any(), any(), any(), any(), any(), any());
|
||||||
|
verify(fixture.jdbc, never()).update(contains("used_bytes = used_bytes +"), any(), any(), any(), any());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void cleanupStatesAreExcludedFromDedupeAndCreateFreshIntent() {
|
||||||
|
Fixture fixture = fixture();
|
||||||
|
stubPhaseOne(fixture, 4L);
|
||||||
|
when(fixture.store.uploadPhysical(eq("personal-private"), anyString(), eq("text/plain"), any(byte[].class)))
|
||||||
|
.thenReturn("https://private.invalid/object");
|
||||||
|
when(fixture.jdbc.update(contains("set o.url ="), anyString(), anyString(), eq(101L), eq("000000"),
|
||||||
|
eq(101L), eq(101L), anyString(), eq(100L), anyString())).thenReturn(1);
|
||||||
|
|
||||||
|
ItemCreatedResponse response = fixture.service.createFile(OWNER,
|
||||||
|
new MockMultipartFile("file", "notes.txt", "text/plain", new byte[]{1, 2, 3, 4}), null, null);
|
||||||
|
|
||||||
|
assertEquals(100L, response.itemId());
|
||||||
|
ArgumentCaptor<String> dedupeSql = ArgumentCaptor.forClass(String.class);
|
||||||
|
verify(fixture.jdbc).queryForList(dedupeSql.capture(), eq("000000"), eq(101L), eq(7L), anyString());
|
||||||
|
assertDedupeStatesAndBinding(dedupeSql.getValue());
|
||||||
|
assertFalse(dedupeSql.getValue().contains("'CLEANING'"));
|
||||||
|
assertFalse(dedupeSql.getValue().contains("'TOMBSTONE'"));
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void uploadFailureClaimsCleaningAndDeletesPhysicalButRetainsDurableIntent() {
|
void uploadFailureClaimsCleaningAndDeletesPhysicalButRetainsDurableIntent() {
|
||||||
Fixture fixture = fixture();
|
Fixture fixture = fixture();
|
||||||
@@ -438,6 +479,13 @@ class PersonalIngestionServiceTest {
|
|||||||
return json;
|
return json;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static void assertDedupeStatesAndBinding(String sql) {
|
||||||
|
assertTrue(sql.contains("in ('PENDING', 'READY')"));
|
||||||
|
assertTrue(sql.contains("$.source')) = 'personal'"));
|
||||||
|
assertTrue(sql.contains("$.itemId')) as unsigned) = i.id"));
|
||||||
|
assertTrue(sql.contains("i.status <> 'DELETED'"));
|
||||||
|
}
|
||||||
|
|
||||||
private record Fixture(JdbcTemplate jdbc, PersonalSpaceService spaces, PersonalObjectStore store,
|
private record Fixture(JdbcTemplate jdbc, PersonalSpaceService spaces, PersonalObjectStore store,
|
||||||
PersonalIngestionService service, TestTransactionManager transactions) {
|
PersonalIngestionService service, TestTransactionManager transactions) {
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user