diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalIngestionWorker.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalIngestionWorker.java index 7d78d117..5f8edf24 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalIngestionWorker.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalIngestionWorker.java @@ -8,6 +8,7 @@ import org.dromara.common.core.exception.ServiceException; import org.dromara.common.oss.core.OssClient; import org.dromara.common.oss.enums.AccessPolicyType; import org.dromara.common.oss.factory.OssFactory; +import org.dromara.common.tenant.helper.TenantHelper; import org.dromara.system.domain.vo.SysOssVo; import org.dromara.system.service.ISysOssService; import org.springframework.beans.factory.annotation.Autowired; @@ -33,6 +34,7 @@ public class PersonalIngestionWorker { private final KnowledgeDocumentParser parser; private final TransactionTemplate transactionTemplate; private final StoredObjectReader objectReader; + private final TenantRunner tenantRunner; private final long maxInputBytes; private final int chunkSize; private final int chunkOverlap; @@ -46,17 +48,18 @@ public class PersonalIngestionWorker { this(jdbcTemplate, parser, new TransactionTemplate(transactionManager), defaultReader(ossService, PersonalIngestionWorker::ossClient), configuredMaxBytes(properties), properties.getChunkSize(), properties.getChunkOverlap(), properties.getParsingLeaseMinutes(), - properties.getMaxParseAttempts()); + properties.getMaxParseAttempts(), PersonalIngestionWorker::runInTenant); } private PersonalIngestionWorker(JdbcTemplate jdbcTemplate, KnowledgeDocumentParser parser, TransactionTemplate transactionTemplate, StoredObjectReader objectReader, long maxInputBytes, int chunkSize, int chunkOverlap, - int parsingLeaseMinutes, int maxParseAttempts) { + int parsingLeaseMinutes, int maxParseAttempts, TenantRunner tenantRunner) { this.jdbcTemplate = jdbcTemplate; this.parser = parser; this.transactionTemplate = transactionTemplate; this.objectReader = objectReader; + this.tenantRunner = tenantRunner; this.maxInputBytes = maxInputBytes; if (chunkSize <= 0 || chunkOverlap < 0 || chunkOverlap >= chunkSize || parsingLeaseMinutes <= 0 || maxParseAttempts <= 0) { @@ -73,7 +76,15 @@ public class PersonalIngestionWorker { TransactionTemplate transactionTemplate, StoredObjectReader objectReader) { return new PersonalIngestionWorker(jdbcTemplate, parser, transactionTemplate, objectReader, - 20L * 1024 * 1024, 800, 120, 15, 3); + 20L * 1024 * 1024, 800, 120, 15, 3, (tenantId, operation) -> operation.execute()); + } + + public static PersonalIngestionWorker forTest(JdbcTemplate jdbcTemplate, ISysOssService ossService, + KnowledgeDocumentParser parser, + TransactionTemplate transactionTemplate, + StoredObjectReader objectReader, TenantRunner tenantRunner) { + return new PersonalIngestionWorker(jdbcTemplate, parser, transactionTemplate, objectReader, + 20L * 1024 * 1024, 800, 120, 15, 3, tenantRunner); } public static StoredObjectReader objectReaderForTest(ISysOssService ossService, @@ -157,8 +168,8 @@ public class PersonalIngestionWorker { int attemptVersion = Math.addExact(item.attemptCount(), 1); try { - StoredObject stored = objectReader.read( - item.ossId(), ownerObjectPrefix(item), item.ownerUserId(), maxInputBytes); + StoredObject stored = tenantRunner.execute(item.tenantId(), () -> objectReader.read( + item.ossId(), ownerObjectPrefix(item), item.ownerUserId(), maxInputBytes)); ParsedDocument document = parser.parse(stored.fileName(), item.mimeType(), stored.bytes()); List chunks = document.chunks(chunkSize, chunkOverlap); if (chunks.isEmpty()) { @@ -345,6 +356,16 @@ public class PersonalIngestionWorker { StoredObject read(long ossId, String expectedPrefix, long ownerUserId, long maxBytes) throws Exception; } + @FunctionalInterface + public interface TenantOperation { + StoredObject execute() throws Exception; + } + + @FunctionalInterface + public interface TenantRunner { + StoredObject execute(String tenantId, TenantOperation operation) throws Exception; + } + @FunctionalInterface public interface OssClientProvider { OssClient get(String configKey); @@ -373,4 +394,15 @@ public class PersonalIngestionWorker { ? OssFactory.instance() : OssFactory.instance(configKey); } + + private static StoredObject runInTenant(String tenantId, TenantOperation operation) throws Exception { + String previous = TenantHelper.getDynamic(); + TenantHelper.setDynamic(tenantId); + try { + return operation.execute(); + } finally { + TenantHelper.clearDynamic(); + if (previous != null && !previous.isBlank()) TenantHelper.setDynamic(previous); + } + } } diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalUrlFetchService.java b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalUrlFetchService.java index b3553547..12f481bc 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalUrlFetchService.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/main/java/org/dromara/aihr/personal/service/PersonalUrlFetchService.java @@ -34,6 +34,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Hashtable; import java.util.HashSet; import java.util.HexFormat; @@ -93,7 +94,9 @@ public class PersonalUrlFetchService { @Autowired public PersonalUrlFetchService(PersonalKnowledgeProperties properties) { - this(properties, new DeadlineDnsResolver(DNS_EXECUTOR, new JndiDnsQuery()), new RawSocketFetcher()); + this(properties, new FallbackResolver( + new DeadlineDnsResolver(DNS_EXECUTOR, new JndiDnsQuery()), + new DeadlineSystemResolver(DNS_EXECUTOR, InetAddress::getAllByName)), new RawSocketFetcher()); } private PersonalUrlFetchService(PersonalKnowledgeProperties properties, Resolver resolver, Fetcher fetcher) { @@ -359,6 +362,69 @@ public class PersonalUrlFetchService { List resolve(String host, long deadlineNanos) throws IOException; } + /** Falls back only when the primary resolver is unavailable; empty or unsafe answers remain fail-closed. */ + static final class FallbackResolver implements Resolver { + private final Resolver primary; + private final Resolver fallback; + + FallbackResolver(Resolver primary, Resolver fallback) { + this.primary = primary; + this.fallback = fallback; + } + + @Override + public List resolve(String host, long deadlineNanos) throws IOException { + try { + return primary.resolve(host, deadlineNanos); + } catch (IOException primaryFailure) { + if (deadlineNanos - System.nanoTime() <= 0) throw primaryFailure; + return fallback.resolve(host, deadlineNanos); + } + } + } + + @FunctionalInterface + interface SystemAddressQuery { + InetAddress[] resolve(String host) throws IOException; + } + + /** Bounds the JVM/system resolver with the same end-to-end deadline used by the fetch. */ + static final class DeadlineSystemResolver implements Resolver { + private final ExecutorService executor; + private final SystemAddressQuery query; + + DeadlineSystemResolver(ExecutorService executor, SystemAddressQuery query) { + this.executor = executor; + this.query = query; + } + + @Override + public List resolve(String host, long deadlineNanos) throws IOException { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0) throw new IOException("resolution deadline exceeded"); + Future future; + try { + future = executor.submit(() -> query.resolve(host)); + } catch (RejectedExecutionException ex) { + throw new IOException("resolution unavailable"); + } + try { + InetAddress[] addresses = future.get(remaining, TimeUnit.NANOSECONDS); + return addresses == null ? List.of() : List.copyOf(Arrays.asList(addresses)); + } catch (TimeoutException ex) { + cancelAndPurge(executor, future); + throw new IOException("resolution deadline exceeded"); + } catch (InterruptedException ex) { + cancelAndPurge(executor, future); + Thread.currentThread().interrupt(); + throw new IOException("resolution interrupted"); + } catch (ExecutionException ex) { + cancelAndPurge(executor, future); + throw new IOException("resolution failed"); + } + } + } + @FunctionalInterface interface DnsQuery { List resolve(String host, int timeoutMillis, int retries) throws NamingException; diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalIngestionWorkerTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalIngestionWorkerTest.java index afb4ecbb..cfadff59 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalIngestionWorkerTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalIngestionWorkerTest.java @@ -20,6 +20,7 @@ import java.sql.Timestamp; import java.time.LocalDateTime; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -36,6 +37,30 @@ import static org.mockito.Mockito.when; @Tag("dev") class PersonalIngestionWorkerTest { + @Test + void workerReadsPrivateObjectInsideItemTenantScope() throws Exception { + JdbcTemplate jdbc = mock(JdbcTemplate.class); + KnowledgeDocumentParser parser = mock(KnowledgeDocumentParser.class); + AtomicReference tenantSeen = new AtomicReference<>(); + when(jdbc.queryForList(contains("status = 'QUEUED'"))).thenReturn(List.of(item())); + when(jdbc.update(contains("status = 'PARSING'"), eq("000000"), eq(101L), eq(9L), eq(0))).thenReturn(1); + when(parser.parse(any(), any(), any(byte[].class))) + .thenThrow(new KnowledgeDocumentParser.ParseException( + KnowledgeDocumentParser.Failure.INVALID, "stop after tenant-scoped read")); + PersonalIngestionWorker worker = PersonalIngestionWorker.forTest( + jdbc, mock(ISysOssService.class), parser, immediateTransactions(), + (ossId, prefix, ownerUserId, maxBytes) -> + new PersonalIngestionWorker.StoredObject("sample.pdf", new byte[]{1}), + (tenantId, operation) -> { + tenantSeen.set(tenantId); + return operation.execute(); + }); + + assertTrue(worker.processNext()); + + assertEquals("000000", tenantSeen.get()); + } + @Test void workerQueueOnlySelectsReadyUploadIntents() { JdbcTemplate jdbc = mock(JdbcTemplate.class); diff --git a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalUrlFetchServiceTest.java b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalUrlFetchServiceTest.java index e33f6dcf..be45e9c9 100644 --- a/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalUrlFetchServiceTest.java +++ b/backend/ruoyi-modules/ruoyi-aihr/src/test/java/org/dromara/aihr/personal/PersonalUrlFetchServiceTest.java @@ -245,6 +245,66 @@ class PersonalUrlFetchServiceTest { assertEquals("0", environment.get("com.sun.jndi.dns.timeout.retries")); } + @Test + void fallsBackToBoundedSystemDnsOnlyWhenJndiResolutionFails() throws Exception { + AtomicInteger fallbackCalls = new AtomicInteger(); + var resolver = new PersonalUrlFetchService.FallbackResolver( + (host, deadline) -> { throw new IOException("JNDI unavailable"); }, + (host, deadline) -> { fallbackCalls.incrementAndGet(); return List.of(PUBLIC); }); + var service = fixture(resolver, request -> ok("text/plain", "ok")); + + assertEquals("https://example.com/", service.validate("https://example.com/").toString()); + assertEquals(1, fallbackCalls.get()); + + fallbackCalls.set(0); + var emptyPrimary = new PersonalUrlFetchService.FallbackResolver( + (host, deadline) -> List.of(), + (host, deadline) -> { fallbackCalls.incrementAndGet(); return List.of(PUBLIC); }); + assertCode("PERSONAL_URL_BLOCKED", () -> fixture(emptyPrimary, request -> ok("text/plain", "ok")) + .validate("https://example.com/"), "empty primary result must fail closed"); + assertEquals(0, fallbackCalls.get()); + } + + @Test + void validatesEveryFallbackAddressAndFailsClosedWhenFallbackFails() { + var privateFallback = new PersonalUrlFetchService.FallbackResolver( + (host, deadline) -> { throw new IOException("JNDI unavailable"); }, + (host, deadline) -> List.of(PUBLIC, address("127.0.0.1"))); + assertCode("PERSONAL_URL_BLOCKED", () -> fixture(privateFallback, request -> ok("text/plain", "ok")) + .validate("https://example.com/"), "mixed fallback addresses"); + + var failedFallback = new PersonalUrlFetchService.FallbackResolver( + (host, deadline) -> { throw new IOException("JNDI unavailable"); }, + (host, deadline) -> { throw new IOException("system DNS unavailable"); }); + assertCode("PERSONAL_URL_BLOCKED", () -> fixture(failedFallback, request -> ok("text/plain", "ok")) + .validate("https://example.com/"), "fallback failure"); + } + + @Test + void boundsSystemDnsFallbackByTheSharedDeadline() throws Exception { + ExecutorService executor = boundedExecutor("system-dns-wall-test"); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try { + var resolver = new PersonalUrlFetchService.DeadlineSystemResolver(executor, host -> { + entered.countDown(); + boolean done = false; + while (!done) { + try { release.await(); done = true; } + catch (InterruptedException ignored) { } + } + return new InetAddress[]{PUBLIC}; + }); + long deadline = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(30); + assertThrows(IOException.class, () -> resolver.resolve("example.com", deadline)); + assertTrue(entered.await(1, TimeUnit.SECONDS)); + } finally { + release.countDown(); + executor.shutdownNow(); + assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + } + } + @Test void outerDnsDeadlineReturnsWhenQueryIgnoresInterrupt() throws Exception { ExecutorService executor = boundedExecutor("dns-wall-test"); diff --git a/docs/DEMO_ACCEPTANCE.md b/docs/DEMO_ACCEPTANCE.md index ccfc1ba3..35c427e2 100644 --- a/docs/DEMO_ACCEPTANCE.md +++ b/docs/DEMO_ACCEPTANCE.md @@ -52,6 +52,6 @@ - 大模型不作为演示硬依赖:模型管理已启用 chat 模型时,三角色对练为真实 LLM 生成与评分(asr/tts 配置后语音输入/播报可用);未配置或现场调用失败时全链路自动回退 seed,演示不中断。 # 个人 AI 助理 P0 验收 -先运行 `./scripts/personal-assistant-smoke.sh`,必须输出 `PASS`。浏览器使用手机号 A 登录后,依次收藏文字、PDF 与公开网页,等待资料状态变为 READY;按采集日期检索,并分别验证个人、企业与 mixed 问答的引用域。删除个人资料后,详情和搜索应立即不可见。 +先运行 `./scripts/personal-assistant-smoke.sh`,必须输出 `PASS`。脚本会真实创建并解析 TEXT、`cupsfilter` PDF 与公开网页(默认 `https://example.com/`),验证三类资料 READY、按 itemIds 可检索、回答引用来自实际命中资料,并在删除后确认 MySQL/MinIO/Qdrant 零残留;公网不可达必须失败,不允许改用 localhost 绕过 SSRF。浏览器使用手机号 A 登录后,依次收藏文字、PDF 与公开网页,等待资料状态变为 READY;按采集日期检索,并分别验证个人、企业与 mixed 问答的引用域。删除个人资料后,详情和搜索应立即不可见。 再使用手机号 B 登录,确认看不到 A 的资料标题、会话与引用,且不能访问 A 的详情、下载、重试或删除接口。私网 URL 与云元数据 URL 必须显示明确的 `PERSONAL_URL_BLOCKED`,回答不得出现无引用内容。企业知识未配置明确授权 allowlist 时,ENTERPRISE/mixed 必须 fail-closed。 diff --git a/docs/DEV_SETUP.md b/docs/DEV_SETUP.md index 141fcb4a..88f1919a 100644 --- a/docs/DEV_SETUP.md +++ b/docs/DEV_SETUP.md @@ -209,4 +209,4 @@ curl -k -s https://peilian.njzhmj.top/h5/ | sed -n '1,20p' ./scripts/personal-assistant-smoke.sh ``` -脚本每次生成唯一 smoke 手机号与 run marker,通过开发短信登录创建 A/B,验证 owner 隔离、READY、SSRF 和幂等删除;退出时只按本次 user/item/session/OSS/job ID 清理。脚本不会输出 token。`./scripts/personal-assistant-smoke.sh --signal-self-test` 可单独验证 INT/TERM 分别返回 130/143。 +脚本每次生成唯一 smoke 手机号与 run marker,通过开发短信登录创建 A/B,并真实采集 TEXT、由 macOS `cupsfilter` 生成的可检索 PDF、公开网页 `https://example.com/`。它会验证三类资料 READY/检索/引用、owner 隔离、私有 OSS 匿名 403、SSRF、幂等删除,以及 MySQL/MinIO/Qdrant 零残留;退出时只按本次 user/item/session/OSS/job ID 与 run marker 回查清理。公开网页可用 `AIHR_PERSONAL_SMOKE_PUBLIC_URL` 覆盖,页面检索词可用 `AIHR_PERSONAL_SMOKE_PUBLIC_QUERY` 覆盖;公网不可达会明确失败,不会退回 localhost 或假数据。脚本不会输出 token。`./scripts/personal-assistant-smoke.sh --signal-self-test` 可单独验证 INT/TERM 分别返回 130/143。 diff --git a/scripts/personal-assistant-smoke.sh b/scripts/personal-assistant-smoke.sh index 92faa63f..6967ef12 100755 --- a/scripts/personal-assistant-smoke.sh +++ b/scripts/personal-assistant-smoke.sh @@ -13,15 +13,27 @@ PHONE_SUFFIX=$(( (10#$(date +%s) + $$ + RANDOM) % 99999998 + 1 )) PHONE_A="138$(printf '%08d' "$PHONE_SUFFIX")" PHONE_B="138$(printf '%08d' $((PHONE_SUFFIX % 99999999 + 1)))" TITLE="personal-assistant-smoke-$RUN_ID" -CONTENT="个人助理隔离烟测唯一标记 ${RUN_ID},物业催费沟通资料。" +TEXT_QUERY="Personal text isolation evidence" +CONTENT="个人助理隔离烟测唯一标记 ${RUN_ID},物业催费沟通资料。${TEXT_QUERY} for run ${RUN_ID}." +TEXT_TITLE="$TITLE-text" +PDF_TITLE="$TITLE-pdf" +URL_TITLE="$TITLE-url" +PDF_QUERY="Personal PDF verification evidence" +PUBLIC_URL="${AIHR_PERSONAL_SMOKE_PUBLIC_URL:-https://example.com/}" +PUBLIC_QUERY="${AIHR_PERSONAL_SMOKE_PUBLIC_QUERY:-Example Domain}" TMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/wygj-personal-smoke.XXXXXX")" chmod 700 "$TMP_ROOT" TOKEN_A="" CLIENT_A="" TOKEN_B="" CLIENT_B="" -USER_A="" USER_B="" ITEM_ID="" OSS_ID="" OBJECT_KEY="" OSS_URL="" JOB_ID="" SESSION_ID="" +USER_A="" USER_B="" SESSION_ID="" OWN_PHONE_A=0 OWN_PHONE_B=0 CLEANED=0 HTTP_STATUS="" HTTP_BODY="" LOGIN_TOKEN="" LOGIN_CLIENT="" TEMP_FILES=() +ITEM_IDS=() +OSS_IDS=() +OBJECT_KEYS=() +OSS_URLS=() +JOB_IDS=() fail() { echo "FAIL: $*" >&2; exit 1; } need() { command -v "$1" >/dev/null 2>&1 || fail "missing command: $1"; } @@ -38,6 +50,38 @@ register_tmp() { TEMP_FILES+=("$path") } +append_item_metadata() { + local item_id="$1" item_title="$2" oss_id object_key oss_url + read -r oss_id object_key oss_url <<<"$(mysql "select o.oss_id,o.file_name,o.url from sys_oss o + join aihr_personal_item i on i.oss_id=o.oss_id + where i.id=$item_id and i.owner_user_id=$USER_A and i.title='$item_title' limit 1")" + [[ "$oss_id" =~ ^[0-9]+$ && "$object_key" =~ ^personal/000000/$USER_A/$item_id/[A-Za-z0-9._/-]+$ \ + && "$object_key" != *".."* && -n "$oss_url" ]] || fail "unexpected personal OSS ownership metadata item=$item_id" + ITEM_IDS+=("$item_id") + OSS_IDS+=("$oss_id") + OBJECT_KEYS+=("$object_key") + OSS_URLS+=("$oss_url") +} + +discover_run_items() { + [[ "$USER_A" =~ ^[0-9]+$ ]] || return 0 + while IFS=$'\t' read -r item_id oss_id object_key oss_url; do + [[ "$item_id" =~ ^[0-9]+$ ]] || continue + local seen=0 existing + for existing in "${ITEM_IDS[@]}"; do + [[ "$existing" == "$item_id" ]] && seen=1 && break + done + [[ "$seen" == 0 ]] || continue + ITEM_IDS+=("$item_id") + OSS_IDS+=("${oss_id:-}") + OBJECT_KEYS+=("${object_key:-}") + OSS_URLS+=("${oss_url:-}") + done < <(mysql "select i.id,coalesce(o.oss_id,''),coalesce(o.file_name,''),coalesce(o.url,'') + from aihr_personal_item i left join sys_oss o on o.oss_id=i.oss_id + where i.tenant_id='000000' and i.owner_user_id=$USER_A and i.title like '$TITLE-%' + order by i.id") +} + cleanup_once() { [[ "$CLEANED" == 0 ]] || return 0 CLEANED=1 @@ -51,36 +95,30 @@ cleanup_once() { if [[ "$OWN_PHONE_B" == 1 && ! "$USER_B" =~ ^[0-9]+$ ]]; then USER_B="$(mysql "select user_id from sys_user where phonenumber='$PHONE_B' and remark='移动端短信自动注册' order by user_id desc limit 1" | head -1)" fi - if [[ "$USER_A" =~ ^[0-9]+$ && ! "$ITEM_ID" =~ ^[0-9]+$ ]]; then - ITEM_ID="$(mysql "select id from aihr_personal_item where tenant_id='000000' and owner_user_id=$USER_A and title='$TITLE' order by id desc limit 1" | head -1)" - fi - if [[ "$USER_A" =~ ^[0-9]+$ && "$ITEM_ID" =~ ^[0-9]+$ ]]; then - if [[ ! "$OSS_ID" =~ ^[0-9]+$ || -z "$OBJECT_KEY" ]]; then - read -r OSS_ID OBJECT_KEY OSS_URL <<<"$(mysql "select o.oss_id,o.file_name,o.url from sys_oss o join aihr_personal_item i on i.oss_id=o.oss_id where i.id=$ITEM_ID and i.owner_user_id=$USER_A and i.title='$TITLE' limit 1")" - fi - [[ "$JOB_ID" =~ ^[0-9]+$ ]] || JOB_ID="$(mysql "select id from aihr_personal_cleanup_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$ITEM_ID order by id desc limit 1" | head -1)" - fi + discover_run_items if [[ "$USER_A" =~ ^[0-9]+$ && ! "$SESSION_ID" =~ ^[0-9]+$ ]]; then SESSION_ID="$(mysql "select s.id from aihr_personal_chat_session s join aihr_personal_chat_message m on m.session_id=s.id and m.owner_user_id=s.owner_user_id where s.tenant_id='000000' and s.owner_user_id=$USER_A and m.content like '%$RUN_ID%' order by s.id desc limit 1" | head -1)" fi - if [[ "$USER_A" =~ ^[0-9]+$ && "$ITEM_ID" =~ ^[0-9]+$ ]]; then + local index item_id oss_id object_key + for index in "${!ITEM_IDS[@]}"; do + item_id="${ITEM_IDS[$index]}" + oss_id="${OSS_IDS[$index]:-}" + object_key="${OBJECT_KEYS[$index]:-}" + [[ "$item_id" =~ ^[0-9]+$ && "$USER_A" =~ ^[0-9]+$ ]] || continue curl -sS -X POST "$QDRANT_URL/collections/$QDRANT_COLLECTION/points/delete?wait=true" \ -H 'Content-Type: application/json' \ - --data "{\"filter\":{\"must\":[{\"key\":\"tenant_id\",\"match\":{\"value\":\"000000\"}},{\"key\":\"owner_user_id\",\"match\":{\"value\":$USER_A}},{\"key\":\"item_id\",\"match\":{\"value\":$ITEM_ID}}]}}" >/dev/null 2>&1 || true - fi - if [[ "$OSS_ID" =~ ^[0-9]+$ && "$USER_A" =~ ^[0-9]+$ && "$ITEM_ID" =~ ^[0-9]+$ \ - && "$OBJECT_KEY" =~ ^personal/000000/$USER_A/$ITEM_ID/[A-Za-z0-9._/-]+$ \ - && "$OBJECT_KEY" != *".."* ]]; then - docker run --rm --network "$DOCKER_NETWORK" \ - -e 'MC_HOST_local=http://ruoyi:ruoyi123@wygj-minio:9000' minio/mc \ - rm --force "local/ruoyi-personal/$OBJECT_KEY" >/dev/null 2>&1 || true - mysql "delete from sys_oss where oss_id=$OSS_ID and service='personal-minio' and file_name='$OBJECT_KEY'" >/dev/null 2>&1 || true - fi - if [[ "$ITEM_ID" =~ ^[0-9]+$ && "$USER_A" =~ ^[0-9]+$ ]]; then - mysql "delete from aihr_personal_fragment where tenant_id='000000' and owner_user_id=$USER_A and item_id=$ITEM_ID; - delete from aihr_personal_cleanup_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$ITEM_ID; - delete from aihr_personal_item where tenant_id='000000' and owner_user_id=$USER_A and id=$ITEM_ID and title='$TITLE';" >/dev/null 2>&1 || true - fi + --data "{\"filter\":{\"must\":[{\"key\":\"tenant_id\",\"match\":{\"value\":\"000000\"}},{\"key\":\"owner_user_id\",\"match\":{\"value\":$USER_A}},{\"key\":\"item_id\",\"match\":{\"value\":$item_id}}]}}" >/dev/null 2>&1 || true + if [[ "$oss_id" =~ ^[0-9]+$ && "$object_key" =~ ^personal/000000/$USER_A/$item_id/[A-Za-z0-9._/-]+$ \ + && "$object_key" != *".."* ]]; then + docker run --rm --network "$DOCKER_NETWORK" \ + -e 'MC_HOST_local=http://ruoyi:ruoyi123@wygj-minio:9000' minio/mc \ + rm --force "local/ruoyi-personal/$object_key" >/dev/null 2>&1 || true + mysql "delete from sys_oss where oss_id=$oss_id and service='personal-minio' and file_name='$object_key'" >/dev/null 2>&1 || true + fi + mysql "delete from aihr_personal_fragment where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id; + delete from aihr_personal_cleanup_job where tenant_id='000000' and owner_user_id=$USER_A and item_id=$item_id; + delete from aihr_personal_item where tenant_id='000000' and owner_user_id=$USER_A and id=$item_id and title like '$TITLE-%';" >/dev/null 2>&1 || true + done if [[ "$SESSION_ID" =~ ^[0-9]+$ && "$USER_A" =~ ^[0-9]+$ ]]; then mysql "delete from aihr_personal_chat_message where tenant_id='000000' and owner_user_id=$USER_A and session_id=$SESSION_ID; delete from aihr_personal_chat_session where tenant_id='000000' and owner_user_id=$USER_A and id=$SESSION_ID;" >/dev/null 2>&1 || true @@ -140,6 +178,17 @@ request() { HTTP_BODY="$(cat "$tmp")" rm -f "$tmp" } +request_file() { + local path="$1" token="$2" client="$3" file="$4" title="$5" tmp + tmp="$(mktemp "$TMP_ROOT/http-file.XXXXXX")" + register_tmp "$tmp" + HTTP_STATUS="$(curl -k -sS -X POST "$API_URL$path" -o "$tmp" -w '%{http_code}' \ + -H "Authorization: Bearer $token" -H "clientid: $client" \ + -F "file=@$file;type=application/pdf;filename=$(basename "$file")" \ + -F "title=$title" || printf '000')" + HTTP_BODY="$(cat "$tmp")" + rm -f "$tmp" +} expect_success() { local label="$1" expect_code "$HTTP_STATUS" 200 "$label HTTP" @@ -162,7 +211,68 @@ login() { LOGIN_CLIENT="$(jq -er '.data.client_id' <<<"$HTTP_BODY")" } -need curl; need jq; need docker; need redis-cli +wait_ready() { + local item_id="$1" label="$2" status="" + for _ in {1..60}; do + request GET "/api/aihr/personal-assistant/items/$item_id" "$TOKEN_A" "$CLIENT_A" + expect_success "$label detail poll" + status="$(jq -r '.data.status // empty' <<<"$HTTP_BODY")" + [[ "$status" == READY ]] && return 0 + [[ "$status" != FAILED ]] || fail "$label processing failed code=$(jq -r '.data.errorCode // empty' <<<"$HTTP_BODY")" + sleep 1 + done + fail "$label did not become READY lastStatus=$status" +} + +assert_anonymous_private() { + local url="$1" label="$2" tmp status + tmp="$(mktemp "$TMP_ROOT/anonymous.XXXXXX")" + register_tmp "$tmp" + status="$(curl -sS -o "$tmp" -w '%{http_code}' "$url" || printf '000')" + expect_code "$status" 403 "$label anonymous GET" +} + +assert_search_hit() { + local item_id="$1" query="$2" expected_type="$3" label="$4" item_ids_json + item_ids_json="$(jq -cn --argjson item "$item_id" '[$item]')" + request POST /api/aihr/personal-assistant/search "$TOKEN_A" "$CLIENT_A" \ + "$(jq -cn --arg query "$query" --argjson ids "$item_ids_json" \ + '{queryText:$query,scope:["PERSONAL"],itemIds:$ids,limit:10}')" + expect_success "$label itemIds search" + [[ "$(jq --arg item "$item_id" --arg type "$expected_type" \ + '[.data.hits[] | select((.itemId|tostring)==$item and .sourceType==$type)]|length' <<<"$HTTP_BODY")" -ge 1 ]] \ + || fail "$label search did not return item=$item_id type=$expected_type query=$query hits=$(jq -c '.data.hits // []' <<<"$HTTP_BODY")" +} + +assert_b_isolation() { + local item_id="$1" label="$2" operation + for operation in detail download retry delete; do + case "$operation" in + detail) request GET "/api/aihr/personal-assistant/items/$item_id" "$TOKEN_B" "$CLIENT_B" ;; + download) request GET "/api/aihr/personal-assistant/items/$item_id/download-url" "$TOKEN_B" "$CLIENT_B" ;; + retry) request POST "/api/aihr/personal-assistant/items/$item_id/retry" "$TOKEN_B" "$CLIENT_B" ;; + delete) request DELETE "/api/aihr/personal-assistant/items/$item_id" "$TOKEN_B" "$CLIENT_B" ;; + esac + expect_error PERSONAL_ITEM_NOT_FOUND "B $operation A $label item" + done + request POST /api/aihr/personal-assistant/search "$TOKEN_B" "$CLIENT_B" \ + "$(jq -cn --argjson item "$item_id" '{queryText:"smoke",scope:["PERSONAL"],itemIds:[$item],limit:10}')" + expect_error PERSONAL_ITEM_NOT_FOUND "B itemIds search A $label isolation" +} + +need curl; need jq; need docker; need redis-cli; need cupsfilter +case "$PUBLIC_URL" in + http://localhost*|https://localhost*|http://127.*|https://127.*|http://\[*|https://\[*|http://169.254.*|https://169.254.*) + fail "AIHR_PERSONAL_SMOKE_PUBLIC_URL must be a public URL, not localhost/private metadata" + ;; + http://*|https://*) ;; + *) fail "AIHR_PERSONAL_SMOKE_PUBLIC_URL must use http or https" ;; +esac +PUBLIC_EFFECTIVE_URL="$(curl --fail --location --silent --show-error --max-time 20 \ + --output /dev/null --write-out '%{url_effective}' "$PUBLIC_URL")" \ + || fail "public URL unreachable: $PUBLIC_URL" +[[ "$PUBLIC_EFFECTIVE_URL" == http://* || "$PUBLIC_EFFECTIVE_URL" == https://* ]] \ + || fail "public URL did not resolve to HTTP(S): $PUBLIC_URL" request GET /auth/tenant/list expect_success "backend health" docker ps --format '{{.Names}}' | grep -qx "$DB_CONTAINER" || fail "database container not running: $DB_CONTAINER" @@ -184,46 +294,64 @@ USER_B="$(mysql "select user_id from sys_user where phonenumber='$PHONE_B' and r [[ "$USER_B" =~ ^[0-9]+$ && "$USER_A" != "$USER_B" ]] || fail "B smoke user was not created independently" request POST /api/aihr/personal-assistant/items/text "$TOKEN_A" "$CLIENT_A" \ - "{\"title\":\"$TITLE\",\"content\":\"$CONTENT\",\"tags\":[\"smoke\",\"$RUN_ID\"]}" + "$(jq -cn --arg title "$TEXT_TITLE" --arg content "$CONTENT" --arg run "$RUN_ID" \ + '{title:$title,content:$content,tags:["smoke",$run]}')" expect_success "A create text item" -ITEM_ID="$(jq -er '.data.itemId | tostring' <<<"$HTTP_BODY")" -read -r OSS_ID OBJECT_KEY OSS_URL <<<"$(mysql "select o.oss_id,o.file_name,o.url from sys_oss o join aihr_personal_item i on i.oss_id=o.oss_id where i.id=$ITEM_ID and i.owner_user_id=$USER_A and i.title='$TITLE'")" -[[ "$OSS_ID" =~ ^[0-9]+$ && "$OBJECT_KEY" =~ ^personal/000000/$USER_A/$ITEM_ID/[A-Za-z0-9._/-]+$ ]] || fail "unexpected personal OSS ownership metadata" +TEXT_ITEM_ID="$(jq -er '.data.itemId | tostring' <<<"$HTTP_BODY")" +append_item_metadata "$TEXT_ITEM_ID" "$TEXT_TITLE" -for _ in {1..40}; do - request GET "/api/aihr/personal-assistant/items/$ITEM_ID" "$TOKEN_A" "$CLIENT_A" - [[ "$(jq -r '.data.status // empty' <<<"$HTTP_BODY")" == READY ]] && break - sleep 1 +PDF_TEXT="$TMP_ROOT/personal-smoke-$RUN_ID.txt" +PDF_FILE="$TMP_ROOT/personal-smoke-$RUN_ID.pdf" +printf 'Personal PDF verification evidence for run %s. Property service fee communication checklist.\n' \ + "$RUN_ID" >"$PDF_TEXT" +cupsfilter -m application/pdf "$PDF_TEXT" >"$PDF_FILE" 2>"$TMP_ROOT/cupsfilter.log" \ + || fail "cupsfilter failed to generate smoke PDF" +[[ -s "$PDF_FILE" && "$(head -c 4 "$PDF_FILE")" == '%PDF' ]] || fail "cupsfilter output is not a PDF" +request_file /api/aihr/personal-assistant/items/file "$TOKEN_A" "$CLIENT_A" "$PDF_FILE" "$PDF_TITLE" +expect_success "A create PDF item" +PDF_ITEM_ID="$(jq -er '.data.itemId | tostring' <<<"$HTTP_BODY")" +append_item_metadata "$PDF_ITEM_ID" "$PDF_TITLE" + +request POST /api/aihr/personal-assistant/items/url "$TOKEN_A" "$CLIENT_A" \ + "$(jq -cn --arg url "$PUBLIC_URL" --arg title "$URL_TITLE" '{url:$url,title:$title}')" +expect_success "A create public URL item" +URL_ITEM_ID="$(jq -er '.data.itemId | tostring' <<<"$HTTP_BODY")" +append_item_metadata "$URL_ITEM_ID" "$URL_TITLE" +expect_code "${#ITEM_IDS[@]}" 3 "three personal items captured" + +wait_ready "$TEXT_ITEM_ID" "A text item" +expect_code "$(jq -r '.data.sourceType' <<<"$HTTP_BODY")" TEXT "A text source type" +wait_ready "$PDF_ITEM_ID" "A PDF item" +expect_code "$(jq -r '.data.sourceType' <<<"$HTTP_BODY")" FILE "A PDF source type" +expect_code "$(jq -r '.data.mimeType' <<<"$HTTP_BODY")" application/pdf "A PDF mime type" +wait_ready "$URL_ITEM_ID" "A public URL item" +expect_code "$(jq -r '.data.sourceType' <<<"$HTTP_BODY")" URL "A URL source type" +expect_code "$(jq -r '.data.originalUrl' <<<"$HTTP_BODY")" "$PUBLIC_EFFECTIVE_URL" "A URL originalUrl" + +for index in "${!OSS_URLS[@]}"; do + assert_anonymous_private "${OSS_URLS[$index]}" "personal object ${ITEM_IDS[$index]}" done -expect_success "A item detail" -expect_code "$(jq -r '.data.status' <<<"$HTTP_BODY")" READY "A text item processing" -anon_tmp="$(mktemp "$TMP_ROOT/anonymous.XXXXXX")"; register_tmp "$anon_tmp" -anon_status="$(curl -sS -o "$anon_tmp" -w '%{http_code}' "$OSS_URL" || printf '000')" -expect_code "$anon_status" 403 "personal object anonymous GET" -request POST /api/aihr/personal-assistant/search "$TOKEN_A" "$CLIENT_A" \ - "{\"queryText\":\"隔离烟测\",\"scope\":[\"PERSONAL\"],\"itemIds\":[\"$ITEM_ID\"],\"limit\":10}" -expect_success "A itemIds search" -[[ "$(jq --arg item "$ITEM_ID" '[.data.hits[] | select((.itemId|tostring)==$item)]|length' <<<"$HTTP_BODY")" -ge 1 ]] || fail "A search did not return own item" +assert_search_hit "$TEXT_ITEM_ID" "$TEXT_QUERY" TEXT text +assert_search_hit "$PDF_ITEM_ID" "$PDF_QUERY" FILE PDF +assert_search_hit "$URL_ITEM_ID" "$PUBLIC_QUERY" URL URL + +ALL_ITEM_IDS_JSON="$(printf '%s\n' "$TEXT_ITEM_ID" "$PDF_ITEM_ID" "$URL_ITEM_ID" | jq -s 'map(tonumber)')" request POST /api/aihr/personal-assistant/ask "$TOKEN_A" "$CLIENT_A" \ - "{\"queryText\":\"隔离烟测资料是什么 $RUN_ID\",\"scope\":[\"PERSONAL\"],\"itemIds\":[\"$ITEM_ID\"]}" + "$(jq -cn --arg query "$TEXT_QUERY $PDF_QUERY $PUBLIC_QUERY $RUN_ID" --argjson ids "$ALL_ITEM_IDS_JSON" \ + '{queryText:$query,scope:["PERSONAL"],itemIds:$ids}')" expect_success "A personal answer" SESSION_ID="$(jq -er '.data.sessionId | tostring' <<<"$HTTP_BODY")" [[ "$SESSION_ID" =~ ^[0-9]+$ ]] || fail "A answer did not persist session" +[[ "$(jq '[.data.citations[] | select(.sourceType=="TEXT")]|length' <<<"$HTTP_BODY")" -ge 1 ]] \ + || fail "A answer citations did not include the selected TEXT item" +[[ "$(jq '[.data.citations[] | select(.sourceType=="FILE" or .sourceType=="URL")]|length' <<<"$HTTP_BODY")" -ge 1 ]] \ + || fail "A answer citations did not include the selected PDF/URL item" request GET "/api/aihr/personal-assistant/sessions/$SESSION_ID" "$TOKEN_B" "$CLIENT_B" expect_error PERSONAL_SESSION_NOT_FOUND "B session isolation" -for operation in detail download retry delete; do - case "$operation" in - detail) request GET "/api/aihr/personal-assistant/items/$ITEM_ID" "$TOKEN_B" "$CLIENT_B" ;; - download) request GET "/api/aihr/personal-assistant/items/$ITEM_ID/download-url" "$TOKEN_B" "$CLIENT_B" ;; - retry) request POST "/api/aihr/personal-assistant/items/$ITEM_ID/retry" "$TOKEN_B" "$CLIENT_B" ;; - delete) request DELETE "/api/aihr/personal-assistant/items/$ITEM_ID" "$TOKEN_B" "$CLIENT_B" ;; - esac - expect_error PERSONAL_ITEM_NOT_FOUND "B $operation A item" -done -request POST /api/aihr/personal-assistant/search "$TOKEN_B" "$CLIENT_B" \ - "{\"queryText\":\"隔离烟测\",\"scope\":[\"PERSONAL\"],\"itemIds\":[\"$ITEM_ID\"],\"limit\":10}" -expect_error PERSONAL_ITEM_NOT_FOUND "B itemIds search isolation" +assert_b_isolation "$TEXT_ITEM_ID" text +assert_b_isolation "$PDF_ITEM_ID" PDF +assert_b_isolation "$URL_ITEM_ID" URL for blocked_url in 'http://127.0.0.1/admin' 'http://169.254.169.254/latest/meta-data/'; do request POST /api/aihr/personal-assistant/items/url "$TOKEN_A" "$CLIENT_A" \ @@ -231,9 +359,47 @@ for blocked_url in 'http://127.0.0.1/admin' 'http://169.254.169.254/latest/meta- expect_error PERSONAL_URL_BLOCKED "SSRF $blocked_url" done -request DELETE "/api/aihr/personal-assistant/items/$ITEM_ID" "$TOKEN_A" "$CLIENT_A"; expect_success "A delete" -JOB_ID="$(jq -er '.data.cleanupJobId | tostring' <<<"$HTTP_BODY")" -request DELETE "/api/aihr/personal-assistant/items/$ITEM_ID" "$TOKEN_A" "$CLIENT_A"; expect_success "A repeat delete" -expect_code "$(jq -r '.data.cleanupJobId | tostring' <<<"$HTTP_BODY")" "$JOB_ID" "idempotent delete job" -request GET "/api/aihr/personal-assistant/items/$ITEM_ID" "$TOKEN_A" "$CLIENT_A"; expect_error PERSONAL_ITEM_NOT_FOUND "A item hidden immediately" -echo "PASS: personal assistant isolation, SSRF, readiness and deletion gates run=$RUN_ID" +for item_id in "$TEXT_ITEM_ID" "$PDF_ITEM_ID" "$URL_ITEM_ID"; do + request DELETE "/api/aihr/personal-assistant/items/$item_id" "$TOKEN_A" "$CLIENT_A" + expect_success "A delete item=$item_id" + JOB_IDS+=("$(jq -er '.data.cleanupJobId | tostring' <<<"$HTTP_BODY")") + request GET "/api/aihr/personal-assistant/items/$item_id" "$TOKEN_A" "$CLIENT_A" + expect_error PERSONAL_ITEM_NOT_FOUND "A item=$item_id hidden immediately" +done +request DELETE "/api/aihr/personal-assistant/items/$TEXT_ITEM_ID" "$TOKEN_A" "$CLIENT_A" +expect_success "A repeat text delete" +expect_code "$(jq -r '.data.cleanupJobId | tostring' <<<"$HTTP_BODY")" "${JOB_IDS[0]}" "idempotent delete job" + +for _ in {1..75}; do + cleanup_done="$(mysql "select count(*) from aihr_personal_cleanup_job + where tenant_id='000000' and owner_user_id=$USER_A and item_id in ($TEXT_ITEM_ID,$PDF_ITEM_ID,$URL_ITEM_ID) + and status='DONE'")" + [[ "$cleanup_done" == 3 ]] && break + sleep 1 +done +expect_code "$cleanup_done" 3 "three cleanup jobs completed" + +cleanup_once +expect_code "$(mysql "select count(*) from aihr_personal_item where tenant_id='000000' and owner_user_id=$USER_A and title like '$TITLE-%'")" 0 "run items residual" +expect_code "$(mysql "select count(*) from aihr_personal_fragment where tenant_id='000000' and owner_user_id=$USER_A and item_id in ($TEXT_ITEM_ID,$PDF_ITEM_ID,$URL_ITEM_ID)")" 0 "run fragments residual" +expect_code "$(mysql "select count(*) from aihr_personal_cleanup_job where tenant_id='000000' and owner_user_id=$USER_A and item_id in ($TEXT_ITEM_ID,$PDF_ITEM_ID,$URL_ITEM_ID)")" 0 "run cleanup jobs residual" +qdrant_collection_exists="$(curl -sS "$QDRANT_URL/collections" \ + | jq -er --arg name "$QDRANT_COLLECTION" '[.result.collections[] | select(.name==$name)]|length')" \ + || fail "Qdrant collection discovery failed" +for index in "${!OSS_IDS[@]}"; do + expect_code "$(mysql "select count(*) from sys_oss where oss_id=${OSS_IDS[$index]}")" 0 "run OSS row ${OSS_IDS[$index]} residual" + if docker run --rm --network "$DOCKER_NETWORK" \ + -e 'MC_HOST_local=http://ruoyi:ruoyi123@wygj-minio:9000' minio/mc \ + stat "local/ruoyi-personal/${OBJECT_KEYS[$index]}" >/dev/null 2>&1; then + fail "run MinIO object ${OBJECT_KEYS[$index]} residual" + fi + qdrant_count=0 + if [[ "$qdrant_collection_exists" -ge 1 ]]; then + qdrant_count="$(curl -sS -X POST "$QDRANT_URL/collections/$QDRANT_COLLECTION/points/count" \ + -H 'Content-Type: application/json' \ + --data "{\"exact\":true,\"filter\":{\"must\":[{\"key\":\"tenant_id\",\"match\":{\"value\":\"000000\"}},{\"key\":\"owner_user_id\",\"match\":{\"value\":$USER_A}},{\"key\":\"item_id\",\"match\":{\"value\":${ITEM_IDS[$index]}}}]}}" \ + | jq -er '.result.count')" || fail "Qdrant residual check failed item=${ITEM_IDS[$index]}" + fi + expect_code "$qdrant_count" 0 "run Qdrant points item=${ITEM_IDS[$index]} residual" +done +echo "PASS: personal assistant TEXT/PDF/public URL isolation, retrieval, privacy and cleanup gates run=$RUN_ID"