fix(aihr): clean video asr snippets
This commit is contained in:
+15
-1
@@ -2214,7 +2214,7 @@ public class AihrSopSeedService {
|
||||
KnowledgeHit first = evidenceHits.get(0);
|
||||
String responseCategory = isBlank(category) ? first.category() : category;
|
||||
List<SnippetResponse> snippets = evidenceHits.stream()
|
||||
.map(hit -> new SnippetResponse("第 " + hit.idx() + " 段:" + hit.title(), hit.content(), hit.fragmentId()))
|
||||
.map(hit -> new SnippetResponse("第 " + hit.idx() + " 段:" + hit.title(), displaySnippetText(hit.content()), hit.fragmentId()))
|
||||
.toList();
|
||||
Map<String, DocResponse> docs = new LinkedHashMap<>();
|
||||
for (KnowledgeHit hit : evidenceHits) {
|
||||
@@ -2272,6 +2272,20 @@ public class AihrSopSeedService {
|
||||
.replaceAll("[^\\p{IsHan}\\p{Alnum}]+", "");
|
||||
}
|
||||
|
||||
static String displaySnippetText(String text) {
|
||||
String value = Optional.ofNullable(text).orElse("").strip();
|
||||
if (!value.contains("【语音转写】")) {
|
||||
return value;
|
||||
}
|
||||
return value
|
||||
.replaceAll("[🎼♪♫♬]+", " ")
|
||||
.replaceAll("(?iu)\\b(?:yeah|the|uh|um)\\b[.!。!?,,\\s]*", " ")
|
||||
.replaceAll("[ぁ-ゟァ-ヿ]+[.!。!?,,\\s]*", " ")
|
||||
.replaceAll("\\s+", " ")
|
||||
.trim()
|
||||
.replace("【语音转写】 ", "【语音转写】\n");
|
||||
}
|
||||
|
||||
static SearchResponse rawHitResponse(String category, String queryText, List<DocResponse> docs, List<SnippetResponse> snippets, List<String> extraCautions) {
|
||||
if (snippets == null || snippets.isEmpty()) {
|
||||
return noEvidenceResponse(queryText, category, extraCautions);
|
||||
|
||||
+15
@@ -65,6 +65,21 @@ public class AihrSopSeedServiceTest {
|
||||
assertEquals(3L, evidence.get(2).fragmentId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Tag("dev")
|
||||
public void displaySnippetTextCleansVideoAsrNoise() {
|
||||
String text = "【语音转写】\n[00:00] 🎼加油。🎼 The。Yeah.う。今天的汇报结束。";
|
||||
|
||||
String cleaned = AihrSopSeedService.displaySnippetText(text);
|
||||
|
||||
assertTrue(cleaned.contains("【语音转写】\n[00:00] 加油。"));
|
||||
assertTrue(cleaned.contains("今天的汇报结束。"));
|
||||
assertFalse(cleaned.contains("🎼"));
|
||||
assertFalse(cleaned.contains("The"));
|
||||
assertFalse(cleaned.contains("Yeah"));
|
||||
assertFalse(cleaned.contains("う"));
|
||||
}
|
||||
|
||||
private static AihrSopSeedService.KnowledgeHit hit(Long fragmentId, String title) {
|
||||
return new AihrSopSeedService.KnowledgeHit(fragmentId, title, "sop", "", "doc-" + fragmentId, "片段内容", 1, 1.0);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user