fix(personal): refine prompt PII detection
This commit is contained in:
+109
-12
@@ -2,26 +2,36 @@ package org.dromara.aihr.personal.service;
|
||||
|
||||
import java.text.Normalizer;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public final class PersonalPromptSanitizer {
|
||||
|
||||
private static final String NUMBER_SEPARATOR = "[.．\\s\\-—–·]";
|
||||
private static final Pattern NUMBER_CANDIDATE = Pattern.compile(
|
||||
"(?<![0-9A-Za-z])\\+?\\d(?:" + NUMBER_SEPARATOR + "*\\d){6,18}(?:" + NUMBER_SEPARATOR
|
||||
+ "*[Xx])?(?![0-9A-Za-z])");
|
||||
private static final Pattern MOBILE = Pattern.compile("1[3-9]\\d{9}");
|
||||
private static final Pattern IDENTITY = Pattern.compile("\\d{17}[0-9Xx]");
|
||||
private static final Pattern FIXED_PHONE = Pattern.compile("0\\d{9,11}");
|
||||
private static final Pattern EMAIL = Pattern.compile(
|
||||
"(?i)(?<![A-Z0-9._%+-])[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}(?![A-Z0-9._%+-])");
|
||||
private static final Pattern LABELED_ADDRESS = Pattern.compile(
|
||||
"(?:地址|住址|家庭住址)[:：\\s]+[^,，;；。\\n]{4,80}");
|
||||
private static final Pattern CHINESE_ADDRESS = Pattern.compile(
|
||||
"(?:[\\p{IsHan}]{2,}(?:省|自治区))?[\\p{IsHan}]{2,}市[\\p{IsHan}]{2,}(?:区|县)"
|
||||
+ "[\\p{IsHan}A-Za-z0-9]{1,30}(?:路|街|巷|道|小区|苑|园)\\d{0,4}号?"
|
||||
+ "[\\p{IsHan}A-Za-z0-9栋幢座单元室房]{0,30}");
|
||||
private static final List<Rule> RULES = List.of(
|
||||
new Rule(Pattern.compile("(?<!\\d)(?:\\+?86[ -]?)?1[3-9](?:[ -]?\\d){9}(?!\\d)"), "[手机号]"),
|
||||
new Rule(Pattern.compile("(?<![0-9A-Za-z])\\d{6}[ -]?\\d{8}[ -]?\\d{3}[0-9Xx](?![0-9A-Za-z])"), "[身份证号]"),
|
||||
new Rule(Pattern.compile("(?<!\\d)0\\d{2,3}[ -]?\\d{7,8}(?:-\\d{1,6})?(?!\\d)"), "[固定电话]"),
|
||||
new Rule(Pattern.compile("(?<!\\d)(?:\\d[ -]?){15,18}\\d(?!\\d)"), "[银行卡号]"),
|
||||
new Rule(Pattern.compile("(?i)(?<![A-Z0-9._%+-])[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}(?![A-Z0-9._%+-])"), "[邮箱]"),
|
||||
new Rule(Pattern.compile("(?:姓名|联系人|业主|客户)[:：\\s]*[\\p{IsHan}]{2,4}"), "[姓名]"),
|
||||
new Rule(EMAIL, "[邮箱]"),
|
||||
new Rule(Pattern.compile("(?:姓名|联系人|业主姓名|客户姓名)\\s*[:：]\\s*[\\p{IsHan}]{2,4}"), "[姓名]"),
|
||||
new Rule(Pattern.compile("(?<![\\p{IsHan}])[\\p{IsHan}]{1,3}(?:先生|女士|师傅|经理|主任|主管)(?![\\p{IsHan}])"), "[姓名称谓]"),
|
||||
new Rule(Pattern.compile("\\d{1,3}(?:栋|幢|座|号楼)(?:\\d{1,3}单元)?(?:\\d{2,4}(?:室|房))?"), "[房号]"),
|
||||
new Rule(Pattern.compile("\\d{1,3}单元\\d{2,4}(?:室|房)"), "[房号]"),
|
||||
new Rule(Pattern.compile("[\\p{IsHan}]{2,4}(?:先生|女士|师傅|经理|主任|主管)"), "[姓名称谓]"),
|
||||
new Rule(Pattern.compile("(?:地址|住址|家庭住址)[:：\\s]*[^,，;；\\n]{4,80}"), "[地址]")
|
||||
new Rule(Pattern.compile("\\d{1,3}单元\\d{2,4}(?:室|房)"), "[房号]")
|
||||
);
|
||||
private static final List<Pattern> RESIDUAL_PATTERNS = List.of(
|
||||
Pattern.compile("(?i)(?:护照(?:号)?|passport)[:：\\s]*[A-Z0-9]{5,20}"),
|
||||
Pattern.compile("(?<!\\d)\\d{10,}(?!\\d)"),
|
||||
Pattern.compile("(?i)[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}")
|
||||
EMAIL
|
||||
);
|
||||
|
||||
private PersonalPromptSanitizer() {
|
||||
@@ -29,6 +39,9 @@ public final class PersonalPromptSanitizer {
|
||||
|
||||
public static String sanitize(String value) {
|
||||
String sanitized = normalize(value);
|
||||
sanitized = LABELED_ADDRESS.matcher(sanitized).replaceAll("[地址]");
|
||||
sanitized = CHINESE_ADDRESS.matcher(sanitized).replaceAll("[地址]");
|
||||
sanitized = maskNumberCandidates(sanitized);
|
||||
for (Rule rule : RULES) {
|
||||
sanitized = rule.pattern().matcher(sanitized).replaceAll(rule.replacement());
|
||||
}
|
||||
@@ -37,7 +50,91 @@ public final class PersonalPromptSanitizer {
|
||||
|
||||
public static boolean containsSensitive(String value) {
|
||||
String normalized = normalize(value);
|
||||
return RESIDUAL_PATTERNS.stream().anyMatch(pattern -> pattern.matcher(normalized).find());
|
||||
if (RESIDUAL_PATTERNS.stream().anyMatch(pattern -> pattern.matcher(normalized).find())) {
|
||||
return true;
|
||||
}
|
||||
Matcher matcher = NUMBER_CANDIDATE.matcher(normalized);
|
||||
while (matcher.find()) {
|
||||
if (classifyNumber(normalized, matcher) != null) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static String maskNumberCandidates(String value) {
|
||||
Matcher matcher = NUMBER_CANDIDATE.matcher(value);
|
||||
StringBuffer output = new StringBuffer();
|
||||
while (matcher.find()) {
|
||||
String replacement = classifyNumber(value, matcher);
|
||||
matcher.appendReplacement(output, replacement == null
|
||||
? Matcher.quoteReplacement(matcher.group()) : Matcher.quoteReplacement(replacement));
|
||||
}
|
||||
matcher.appendTail(output);
|
||||
return output.toString();
|
||||
}
|
||||
|
||||
private static String classifyNumber(String value, Matcher matcher) {
|
||||
String compact = matcher.group().replaceAll(NUMBER_SEPARATOR, "");
|
||||
if (compact.startsWith("+")) {
|
||||
compact = compact.substring(1);
|
||||
}
|
||||
String domestic = compact.startsWith("86") && compact.length() == 13 ? compact.substring(2) : compact;
|
||||
String context = value.substring(Math.max(0, matcher.start() - 12), matcher.start());
|
||||
if (MOBILE.matcher(domestic).matches()) {
|
||||
return "[手机号]";
|
||||
}
|
||||
if (IDENTITY.matcher(compact).matches()
|
||||
&& (hasDirectContext(context, "身份证", "证件号", "身份号码") || validIdentityChecksum(compact))) {
|
||||
return "[身份证号]";
|
||||
}
|
||||
if (compact.chars().allMatch(Character::isDigit) && compact.length() >= 16 && compact.length() <= 19
|
||||
&& (hasDirectContext(context, "银行卡", "银行卡号", "银行账号", "卡号") || validLuhn(compact))) {
|
||||
return "[银行卡号]";
|
||||
}
|
||||
if (compact.chars().allMatch(Character::isDigit)
|
||||
&& (FIXED_PHONE.matcher(compact).matches()
|
||||
|| hasDirectContext(context, "手机", "电话", "联系方式", "联系电话"))) {
|
||||
return "[固定电话]";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static boolean hasDirectContext(String context, String... labels) {
|
||||
String trimmed = context.stripTrailing();
|
||||
if (trimmed.endsWith(":") || trimmed.endsWith("：")) {
|
||||
trimmed = trimmed.substring(0, trimmed.length() - 1).stripTrailing();
|
||||
}
|
||||
for (String label : labels) {
|
||||
if (trimmed.endsWith(label)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static boolean validLuhn(String digits) {
|
||||
int sum = 0;
|
||||
boolean doubled = false;
|
||||
for (int i = digits.length() - 1; i >= 0; i--) {
|
||||
int digit = digits.charAt(i) - '0';
|
||||
if (doubled && (digit *= 2) > 9) {
|
||||
digit -= 9;
|
||||
}
|
||||
sum += digit;
|
||||
doubled = !doubled;
|
||||
}
|
||||
return sum % 10 == 0;
|
||||
}
|
||||
|
||||
private static boolean validIdentityChecksum(String identity) {
|
||||
int[] weights = {7, 9, 10, 5, 8, 4, 2, 1, 6, 3, 7, 9, 10, 5, 8, 4, 2};
|
||||
char[] checks = {'1', '0', 'X', '9', '8', '7', '6', '5', '4', '3', '2'};
|
||||
int sum = 0;
|
||||
for (int i = 0; i < weights.length; i++) {
|
||||
sum += (identity.charAt(i) - '0') * weights[i];
|
||||
}
|
||||
return Character.toUpperCase(identity.charAt(17)) == checks[sum % 11];
|
||||
}
|
||||
|
||||
private static String normalize(String value) {
|
||||
|
||||
+32
@@ -198,6 +198,38 @@ class PersonalAnswerServiceTest {
|
||||
assertEquals(sensitive, response.citations().get(0).excerpt());
|
||||
}
|
||||
|
||||
@Test
|
||||
void removesUnicodeSeparatedPiiFromFinalPromptAndPreservesPropertyBusinessMeaning() {
|
||||
List<String> prompts = new ArrayList<>();
|
||||
AtomicInteger modelCalls = new AtomicInteger();
|
||||
String query = "业主投诉漏水，联系人电话138.0000.0000，工单编号202607120001需跟进";
|
||||
String source = "客户反馈很好，南京市鼓楼区银城街12号张三家，备用手机138—0000—0000";
|
||||
PersonalAnswerService service = PersonalAnswerService.forTest(
|
||||
(owner, request) -> List.of(personalHit("1", "正常物业记录", source)),
|
||||
(owner, text, grant, limit) -> List.of(),
|
||||
(system, user, temperature) -> {
|
||||
modelCalls.incrementAndGet();
|
||||
prompts.add(user);
|
||||
return result("答案");
|
||||
},
|
||||
new RecordingPersistence()
|
||||
);
|
||||
|
||||
service.ask(OWNER, new AskRequest(null, query, List.of(SearchScope.PERSONAL),
|
||||
null, null, List.of(), "ANSWER"));
|
||||
|
||||
assertEquals(1, modelCalls.get());
|
||||
String prompt = prompts.get(0);
|
||||
assertFalse(prompt.contains("138.0000.0000"));
|
||||
assertFalse(prompt.contains("138—0000—0000"));
|
||||
assertFalse(prompt.contains("南京市鼓楼区银城街12号张三家"));
|
||||
assertTrue(prompt.contains("[手机号]"));
|
||||
assertTrue(prompt.contains("[地址]"));
|
||||
assertTrue(prompt.contains("业主投诉漏水"));
|
||||
assertTrue(prompt.contains("客户反馈很好"));
|
||||
assertTrue(prompt.contains("工单编号202607120001需跟进"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void residualSensitiveContentFailsClosedWithoutCallingExternalModel() {
|
||||
AtomicInteger modelCalls = new AtomicInteger();
|
||||
|
||||
+22
-3
@@ -4,6 +4,7 @@ import org.dromara.aihr.personal.service.PersonalPromptSanitizer;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@@ -32,12 +33,30 @@ class PersonalPromptSanitizerTest {
|
||||
@Test
|
||||
void normalizesUnicodeAndDoesNotMaskOrdinaryBusinessText() {
|
||||
assertTrue(PersonalPromptSanitizer.sanitize("手机１３８－００００－００００").contains("[手机号]"));
|
||||
String ordinary = "本周完成2026年7月12日收费标准复核，工单编号A12345，计划覆盖3个项目。";
|
||||
assertEquals("手机[手机号]", PersonalPromptSanitizer.sanitize("手机138.0000.0000"));
|
||||
assertEquals("手机[手机号]", PersonalPromptSanitizer.sanitize("手机138—0000—0000"));
|
||||
for (String mobile : new String[]{"138．0000．0000", "138 0000 0000", "138-0000-0000",
|
||||
"138–0000–0000", "138·0000·0000"}) {
|
||||
assertEquals("手机[手机号]", PersonalPromptSanitizer.sanitize("手机" + mobile));
|
||||
}
|
||||
String ordinary = "业主投诉漏水，客户反馈很好，联系人电话待补，工单编号202607120001需跟进。";
|
||||
String sanitized = PersonalPromptSanitizer.sanitize(ordinary);
|
||||
assertTrue(sanitized.contains("收费标准复核"));
|
||||
assertTrue(sanitized.contains("工单编号A12345"));
|
||||
assertTrue(sanitized.contains("业主投诉漏水"));
|
||||
assertTrue(sanitized.contains("客户反馈很好"));
|
||||
assertTrue(sanitized.contains("联系人电话"));
|
||||
assertTrue(sanitized.contains("工单编号202607120001需跟进"));
|
||||
assertFalse(sanitized.contains("["));
|
||||
assertFalse(PersonalPromptSanitizer.containsSensitive(ordinary));
|
||||
assertTrue(PersonalPromptSanitizer.containsSensitive("护照号 E12345678"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void masksUnlabelledChineseAddressWithoutTreatingBusinessNounsAsNames() {
|
||||
String sanitized = PersonalPromptSanitizer.sanitize("南京市鼓楼区银城街12号张三家");
|
||||
|
||||
assertEquals("[地址]", sanitized);
|
||||
assertFalse(sanitized.contains("张三"));
|
||||
assertEquals("南京市鼓楼区收费标准已更新",
|
||||
PersonalPromptSanitizer.sanitize("南京市鼓楼区收费标准已更新"));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user