From 8925bd1cd4f3c488e29569eca5456a290fc7a8e8 Mon Sep 17 00:00:00 2001 From: Symb0x76 <9667434@qq.com> Date: Tue, 14 Oct 2025 15:11:55 +0800 Subject: [PATCH] fix(voice file): introducing copylock to improve accessibility of vocie files --- TODO.md | 4 ++ internal/chatlog/http/route.go | 122 +++++++++------------------------ pkg/filecopy/filecopy.go | 30 ++++---- pkg/util/silk/silk.go | 117 +++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 101 deletions(-) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..00fc82e --- /dev/null +++ b/TODO.md @@ -0,0 +1,4 @@ +# TODO + +- [ ] 修复 html 渲染部分头像缺失的问题 +- [ ] 美化搜索界面使风格一致 diff --git a/internal/chatlog/http/route.go b/internal/chatlog/http/route.go index 2a4e7d2..fef22af 100644 --- a/internal/chatlog/http/route.go +++ b/internal/chatlog/http/route.go @@ -906,7 +906,9 @@ func (s *Service) handleSearch(c *gin.Context) { resp.Offset = offset format := strings.ToLower(strings.TrimSpace(params.Format)) - terms := extractQueryTerms(resp.Query) + if format == "" { + format = "json" + } switch format { case "html": @@ -966,9 +968,9 @@ func (s *Service) handleSearch(c *gin.Context) { c.Writer.WriteString("") c.Writer.WriteString("
时间：" + template.HTMLEscapeString(msg.Time.Format("2006-01-02 15:04:05")) + "发送者：" + template.HTMLEscapeString(senderDisplay) + "
") if snippet := strings.TrimSpace(hit.Snippet); snippet != "" { - h := highlightPlainTextToHTML(snippet, terms) - h = strings.ReplaceAll(h, "\n", "
") - c.Writer.WriteString("
" + h + "
") + escaped := template.HTMLEscapeString(snippet) + escaped = strings.ReplaceAll(escaped, "\n", "
") + c.Writer.WriteString("
" + escaped + "
") } c.Writer.WriteString("
" + messageHTMLPlaceholder(msg) + "
") c.Writer.WriteString("") @@ -1055,7 +1057,7 @@ func (s *Service) handleSearch(c *gin.Context) { } csvWriter.Flush() return - case "json", "": + case "json": c.JSON(http.StatusOK, resp) return default: @@ -1091,6 +1093,11 @@ func (s *Service) handleChatlog(c *gin.Context) { q.Offset = 0 } + format := strings.ToLower(strings.TrimSpace(q.Format)) + if format == "" { + format = "json" + } + // 1. 未指定 talker: 分组输出 if q.Talker == "" { sessionsResp, err := s.db.GetSessions("", 0, 0) @@ -1111,7 +1118,7 @@ func (s *Service) handleChatlog(c *gin.Context) { } groups = append(groups, &grouped{Talker: sess.UserName, TalkerName: sess.NickName, Messages: msgs}) } - switch strings.ToLower(q.Format) { + switch format { case "html": c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") c.Writer.WriteString("Chatlog") @@ -1156,7 +1163,7 @@ func (s *Service) handleChatlog(c *gin.Context) { } } csvWriter.Flush() - default: + case "text", "plain": c.Writer.Header().Set("Content-Type", "text/plain; charset=utf-8") c.Writer.Header().Set("Cache-Control", "no-cache") c.Writer.Header().Set("Connection", "keep-alive") @@ -1179,6 +1186,8 @@ func (s *Service) handleChatlog(c *gin.Context) { } c.Writer.WriteString("-----------------------------\n") } + default: + c.JSON(http.StatusOK, groups) } return } @@ -1189,7 +1198,7 @@ func (s *Service) handleChatlog(c *gin.Context) { errors.Err(c, err) return } - switch strings.ToLower(q.Format) { + switch format { case "html": c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") c.Writer.WriteString("Chatlog") @@ -1260,7 +1269,10 @@ func (s *Service) handleContacts(c *gin.Context) { return } - format := strings.ToLower(q.Format) + format := strings.ToLower(strings.TrimSpace(q.Format)) + if format == "" { + format = "json" + } switch format { case "html": c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") @@ -1375,7 +1387,10 @@ func (s *Service) handleChatRooms(c *gin.Context) { errors.Err(c, err) return } - format := strings.ToLower(q.Format) + format := strings.ToLower(strings.TrimSpace(q.Format)) + if format == "" { + format = "json" + } switch format { case "json": // json @@ -1419,7 +1434,10 @@ func (s *Service) handleSessions(c *gin.Context) { errors.Err(c, err) return } - format := strings.ToLower(q.Format) + format := strings.ToLower(strings.TrimSpace(q.Format)) + if format == "" { + format = "json" + } switch format { case "html": c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") @@ -1537,7 +1555,10 @@ func (s *Service) handleDiary(c *gin.Context) { groups = append(groups, &grouped{Talker: sess.UserName, TalkerName: sess.NickName, Messages: msgs}) } - format := strings.ToLower(q.Format) + format := strings.ToLower(strings.TrimSpace(q.Format)) + if format == "" { + format = "json" + } switch format { case "html": c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") @@ -1812,8 +1833,7 @@ func (s *Service) HandleVoice(c *gin.Context, data []byte) { // 统一占位符：将 PlainTextContent 里形如 ![标签](url) 或 [标签](url) 的模式转成超链接形式，仅显示 [标签]。 var ( - placeholderPattern = regexp.MustCompile(`!?\[([^\]]+)\]\((https?://[^)]+)\)`) - searchTermSplitPattern = regexp.MustCompile(`[\s,，；;、]+`) + placeholderPattern = regexp.MustCompile(`!?\[([^\]]+)\]\((https?://[^)]+)\)`) ) func messageHTMLPlaceholder(m *model.Message) string { @@ -1857,77 +1877,3 @@ func messageHTMLPlaceholder(m *model.Message) string { return anchor }) } - -func extractQueryTerms(raw string) []string { - raw = strings.TrimSpace(raw) - if raw == "" { - return nil - } - parts := searchTermSplitPattern.Split(raw, -1) - terms := make([]string, 0, len(parts)) - seen := make(map[string]struct{}) - for _, part := range parts { - term := strings.Trim(part, "\"'()[]{}") - if term == "" { - continue - } - switch strings.ToUpper(term) { - case "AND", "OR", "NOT": - continue - } - key := strings.ToLower(term) - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - terms = append(terms, term) - } - return terms -} - -func highlightPlainTextToHTML(text string, terms []string) string { - if text == "" { - return "" - } - if len(terms) == 0 { - return template.HTMLEscapeString(text) - } - unique := make(map[string]struct{}) - escaped := make([]string, 0, len(terms)) - for _, term := range terms { - trimmed := strings.TrimSpace(term) - if trimmed == "" { - continue - } - key := strings.ToLower(trimmed) - if _, ok := unique[key]; ok { - continue - } - unique[key] = struct{}{} - escaped = append(escaped, regexp.QuoteMeta(trimmed)) - } - if len(escaped) == 0 { - return template.HTMLEscapeString(text) - } - sort.Slice(escaped, func(i, j int) bool { return len(escaped[i]) > len(escaped[j]) }) - pattern := strings.Join(escaped, "|") - re, err := regexp.Compile("(?i)" + pattern) - if err != nil { - return template.HTMLEscapeString(text) - } - matches := re.FindAllStringIndex(text, -1) - if len(matches) == 0 { - return template.HTMLEscapeString(text) - } - var builder strings.Builder - last := 0 - for _, loc := range matches { - builder.WriteString(template.HTMLEscapeString(text[last:loc[0]])) - builder.WriteString(``) - builder.WriteString(template.HTMLEscapeString(text[loc[0]:loc[1]])) - builder.WriteString("") - last = loc[1] - } - builder.WriteString(template.HTMLEscapeString(text[last:])) - return builder.String() -} diff --git a/pkg/filecopy/filecopy.go b/pkg/filecopy/filecopy.go index 5c0524a..b8d539b 100644 --- a/pkg/filecopy/filecopy.go +++ b/pkg/filecopy/filecopy.go @@ -57,6 +57,7 @@ type FileCopyManager struct { cancel context.CancelFunc // Cancel function for graceful shutdown wg sync.WaitGroup // WaitGroup for goroutine synchronization cacheSize int64 // Current number of cached entries (atomic) + copyLocks sync.Map // Per-cacheKey mutexes to avoid duplicate copies } // FileIndexEntry represents an indexed temporary file with comprehensive metadata. @@ -608,30 +609,28 @@ func (fm *FileCopyManager) GetTempCopy(originalPath string) (string, error) { expectedDataHash = fmt.Sprintf("%x", currentSize+currentModTime.UnixNano())[:16] } - // Strategy 1: Check index for existing file using unified cache key baseName := fm.extractBaseName(originalPath) ext := extractFileExtension(originalPath) // Use unified cache key generation cacheKey := fm.generateCacheKey(fm.instanceID, baseName, ext, currentHash, expectedDataHash) + lock := fm.acquireCopyLock(cacheKey) + lock.Lock() + defer lock.Unlock() + var oldTempPath string // Track old file to delete after successful creation if value, exists := fm.fileIndex.Load(cacheKey); exists { entry := value.(*FileIndexEntry) - // Found cached file, verify it still exists and matches if _, err := os.Stat(entry.TempPath); err == nil && currentSize == entry.Size { - // File exists and size matches, reuse cached copy - entry.SetLastAccess(now) // Update access time atomically - entry.SetOriginalPath(originalPath) // Update original path thread-safely + entry.SetLastAccess(now) + entry.SetOriginalPath(originalPath) return entry.TempPath, nil - } else { - // Cached file is missing or size mismatch, remove from index - fm.fileIndex.Delete(cacheKey) - atomic.AddInt64(&fm.cacheSize, -1) - if err == nil { - // File exists but size mismatch, mark for cleanup - oldTempPath = entry.TempPath - } + } + fm.fileIndex.Delete(cacheKey) + atomic.AddInt64(&fm.cacheSize, -1) + if err == nil { + oldTempPath = entry.TempPath } } @@ -677,6 +676,11 @@ func (fm *FileCopyManager) GetTempCopy(originalPath string) (string, error) { return tempPath, nil } +func (fm *FileCopyManager) acquireCopyLock(cacheKey string) *sync.Mutex { + lockIface, _ := fm.copyLocks.LoadOrStore(cacheKey, &sync.Mutex{}) + return lockIface.(*sync.Mutex) +} + // generateTempPath creates a unique temporary file path using a structured naming convention. // The format is: instanceID_+baseName_+ext_+pathHash_+dataHash.ext // This naming scheme uses "_+" separator to avoid conflicts with filenames containing underscores. diff --git a/pkg/util/silk/silk.go b/pkg/util/silk/silk.go index b7321f1..98f5911 100644 --- a/pkg/util/silk/silk.go +++ b/pkg/util/silk/silk.go @@ -1,17 +1,44 @@ package silk import ( + "bytes" + "compress/gzip" + "compress/zlib" "encoding/binary" "fmt" + "io" + "github.com/rs/zerolog/log" "github.com/sjzar/go-lame" "github.com/sjzar/go-silk" + + "github.com/sjzar/chatlog/pkg/util/zstd" ) const decodedSampleRate = 24000 // Silk2PCM16 解码 Silk 数据，返回 16-bit PCM 采样数据及采样率。 func Silk2PCM16(data []byte) ([]int16, int, error) { + samples, rate, err := decodeSilk(data) + if err == nil { + return samples, rate, nil + } + + log.Debug().Err(err).Msg("silk decode failed, retry with normalized payload") + normalized := normalizeSilkPayload(data) + if bytes.Equal(normalized, data) { + log.Error().Err(err).Msg("silk decode failed; payload unchanged after normalization") + return nil, 0, err + } + + samples, rate, err = decodeSilk(normalized) + if err != nil { + log.Error().Err(err).Msg("silk decode failed after normalization") + } + return samples, rate, err +} + +func decodeSilk(data []byte) ([]int16, int, error) { sd := silk.SilkInit() defer sd.Close() @@ -31,6 +58,96 @@ func Silk2PCM16(data []byte) ([]int16, int, error) { return samples, decodedSampleRate, nil } +var ( + silkMagic = []byte("#!SILK") + zstdMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} +) + +func normalizeSilkPayload(data []byte) []byte { + current := data + for i := 0; i < 3; i++ { + trimmed := bytes.TrimLeft(current, "\x00\xff") + if idx := bytes.Index(trimmed, silkMagic); idx >= 0 { + return trimmed[idx:] + } + + if next, ok := tryDecompress(trimmed); ok { + current = next + continue + } + + if !bytes.Equal(trimmed, current) { + current = trimmed + continue + } + + break + } + return current +} + +type decompressor struct { + name string + match func([]byte) bool + fn func([]byte) ([]byte, error) +} + +var decompressors = []decompressor{ + { + name: "zstd", + match: func(b []byte) bool { + return len(b) >= 4 && bytes.Equal(b[:4], zstdMagic) + }, + fn: func(b []byte) ([]byte, error) { + return zstd.Decompress(b) + }, + }, + { + name: "gzip", + match: func(b []byte) bool { + return len(b) >= 2 && b[0] == 0x1f && b[1] == 0x8b + }, + fn: func(b []byte) ([]byte, error) { + reader, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + defer reader.Close() + return io.ReadAll(reader) + }, + }, + { + name: "zlib", + match: func(b []byte) bool { + return len(b) >= 2 && b[0] == 0x78 + }, + fn: func(b []byte) ([]byte, error) { + reader, err := zlib.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + defer reader.Close() + return io.ReadAll(reader) + }, + }, +} + +func tryDecompress(data []byte) ([]byte, bool) { + for _, dc := range decompressors { + if !dc.match(data) { + continue + } + out, err := dc.fn(data) + if err != nil { + log.Debug().Str("codec", dc.name).Err(err).Msg("silk payload decompress failed") + continue + } + log.Debug().Str("codec", dc.name).Msg("silk payload decompressed") + return out, true + } + return nil, false +} + func Silk2MP3(data []byte) ([]byte, error) { sd := silk.SilkInit()