feat(search): FTS support for wechatdb

This commit is contained in:
Symb0x76
2025-10-08 11:39:34 +08:00
parent 4a170cb33c
commit 293ab54728
17 changed files with 2100 additions and 1214 deletions
+48 -129
View File
@@ -1,156 +1,75 @@
BINARY_NAME := chatlog
GO := go
# Embed version into binary
ifeq ($(VERSION),)
VERSION := $(shell git describe --tags --always --dirty="-dev" 2>/dev/null)
endif
ifeq ($(VERSION),)
VERSION := dev
VERSION := $(shell git describe --tags --always --dirty="-dev")
endif
LDFLAGS := -ldflags '-X "github.com/sjzar/chatlog/pkg/version.Version=$(VERSION)" -w -s'
PLATFORMS := \
darwin/amd64 \
darwin/arm64 \
linux/amd64 \
linux/arm64 \
windows/amd64 \
windows/arm64
darwin/amd64 \
darwin/arm64 \
linux/amd64 \
linux/arm64 \
windows/amd64 \
windows/arm64
UPX_PLATFORMS := \
darwin/amd64 \
linux/amd64 \
linux/arm64 \
windows/amd64
darwin/amd64 \
linux/amd64 \
linux/arm64 \
windows/amd64
# Cross-compiler hints (override if paths differ)
CC_WIN_AMD64 ?= x86_64-w64-mingw32-gcc
CC_WIN_ARM64 ?= aarch64-w64-mingw32-gcc
CC_DARWIN_AMD64 ?= o64-clang
CC_DARWIN_ARM64 ?= oa64-clang
ifeq ($(OS),Windows_NT)
MKDIR_BIN := powershell -NoProfile -Command "New-Item -ItemType Directory -Force -Path 'bin' | Out-Null"
CLEAN_BIN := powershell -NoProfile -Command "if (Test-Path 'bin') { Remove-Item -Recurse -Force 'bin' }"
BINARY_SUFFIX := .exe
else
MKDIR_BIN := mkdir -p bin
CLEAN_BIN := rm -rf bin
BINARY_SUFFIX :=
endif
.PHONY: all clean lint tidy test build upx
.PHONY: all clean lint tidy test build crossbuild upx
all: clean lint tidy test build
clean:
@echo "🧹 Cleaning..."
@rm -rf bin/
@echo "Cleaning..."
@$(CLEAN_BIN)
lint:
@echo "🕵️‍♂️ Running linters..."
@echo "Running linters..."
golangci-lint run ./...
tidy:
@echo "🧼 Tidying up dependencies..."
@echo "Tidying up dependencies..."
$(GO) mod tidy
test:
@echo "🧪 Running tests..."
@echo "Running tests..."
$(GO) test ./... -cover
build:
@echo "🔨 Building for current platform..."
@mkdir -p bin
@CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o bin/$(BINARY_NAME) main.go
@echo "Building for current platform..."
@$(MKDIR_BIN)
@CGO_ENABLED=1 $(GO) build --tags "fts5" -trimpath $(LDFLAGS) -o bin/$(BINARY_NAME)$(BINARY_SUFFIX) main.go
@echo "Done!"
buildcross:
@echo "[cross] Interactive cross-compilation selection (Windows/macOS)"
@mkdir -p bin
@printf "\nSelect target platforms to build (enter digits, e.g., 134):\n 1) windows/amd64\n 2) windows/arm64\n 3) darwin/amd64 (macOS Intel)\n 4) darwin/arm64 (Apple Silicon)\n> "; \
read choices; \
opts=$$(printf "%s" "$$choices" | tr -cd '1234' | sed 's/./& /g'); \
for ch in $$opts; do \
case $$ch in \
1) \
ccbin=$${CC_WIN_AMD64:-x86_64-w64-mingw32-gcc}; \
: # On Windows host: do not force CC, use local toolchain \
if [ "$(OS)" = "Windows_NT" ]; then \
echo "🧭 Host: Windows"; \
tmp="bin/$(BINARY_NAME)_windows_amd64.exe.new.$$RANDOM"; \
GOOS=windows GOARCH=amd64 CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o "$$tmp" main.go && \
{ mv -f "$$tmp" "bin/$(BINARY_NAME)_windows_amd64.exe" 2>/dev/null || \
echo "⚠️ Target locked, kept: $$tmp"; }; \
else \
if command -v "$$ccbin" >/dev/null 2>&1; then \
echo "⚙️ Using CC=$$ccbin"; \
tmp="bin/$(BINARY_NAME)_windows_amd64.exe.new.$$RANDOM"; \
env CC="$$ccbin" GOOS=windows GOARCH=amd64 CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o "$$tmp" main.go && \
{ mv -f "$$tmp" "bin/$(BINARY_NAME)_windows_amd64.exe" 2>/dev/null || \
echo "⚠️ Target locked, kept: $$tmp"; }; \
else \
echo "⏭️ Skip windows/amd64: $$ccbin not found (Arch: sudo pacman -S --needed mingw-w64-gcc)"; \
fi; \
fi \
;; \
2) \
ccbin=$${CC_WIN_ARM64:-aarch64-w64-mingw32-gcc}; \
: # windows/arm64 needs aarch64 toolchain \
if [ "$(OS)" = "Windows_NT" ]; then \
if command -v "$$ccbin" >/dev/null 2>&1; then \
echo "⚙️ Using CC=$$ccbin"; \
tmp="bin/$(BINARY_NAME)_windows_arm64.exe.new.$$RANDOM"; \
env CC="$$ccbin" GOOS=windows GOARCH=arm64 CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o "$$tmp" main.go && \
{ mv -f "$$tmp" "bin/$(BINARY_NAME)_windows_arm64.exe" 2>/dev/null || \
echo "⚠️ Target locked, kept: $$tmp"; }; \
else \
echo "⏭️ Skip windows/arm64: $$ccbin not found (MSYS2: pacman -S mingw-w64-aarch64-gcc)"; \
fi; \
else \
if command -v "$$ccbin" >/dev/null 2>&1; then \
echo "⚙️ Using CC=$$ccbin"; \
tmp="bin/$(BINARY_NAME)_windows_arm64.exe.new.$$RANDOM"; \
env CC="$$ccbin" GOOS=windows GOARCH=arm64 CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o "$$tmp" main.go && \
{ mv -f "$$tmp" "bin/$(BINARY_NAME)_windows_arm64.exe" 2>/dev/null || \
echo "⚠️ Target locked, kept: $$tmp"; }; \
else \
echo "⏭️ Skip windows/arm64: $$ccbin not found (Arch: sudo pacman -S --needed mingw-w64-gcc)"; \
fi; \
fi \
;; \
3) \
ccdarwin=$${CC_DARWIN_AMD64:-o64-clang}; \
if [ "$(OS)" = "Windows_NT" ] || [ "$$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]')" = "linux" ]; then \
: # Non-macOS hosts need osxcross o64-clang \
if command -v "$$ccdarwin" >/dev/null 2>&1; then \
echo "⚙️ Using CC=$$ccdarwin"; \
env CC="$$ccdarwin" GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o bin/$(BINARY_NAME)_darwin_amd64; \
else \
echo "⏭️ Skip darwin/amd64: $$ccdarwin not found (need osxcross)"; \
fi; \
else \
: # macOS host: try system clang \
if command -v clang >/dev/null 2>&1; then \
echo "⚙️ Using CC=clang"; \
env CC=clang GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o bin/$(BINARY_NAME)_darwin_amd64; \
else \
echo "⏭️ Skip darwin/amd64: clang not found"; \
fi; \
fi \
;; \
4) \
ccdarwin=$${CC_DARWIN_ARM64:-oa64-clang}; \
if [ "$(OS)" = "Windows_NT" ] || [ "$$(uname -s 2>/dev/null | tr '[:upper:]' '[:lower:]')" = "linux" ]; then \
: # Non-macOS hosts need osxcross oa64-clang \
if command -v "$$ccdarwin" >/dev/null 2>&1; then \
echo "⚙️ Using CC=$$ccdarwin"; \
env CC="$$ccdarwin" GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o bin/$(BINARY_NAME)_darwin_arm64; \
else \
echo "⏭️ Skip darwin/arm64: $$ccdarwin not found (need osxcross)"; \
fi; \
else \
: # macOS host: try system clang \
if command -v clang >/dev/null 2>&1; then \
echo "⚙️ Using CC=clang"; \
env CC=clang GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 $(GO) build -trimpath $(LDFLAGS) -o bin/$(BINARY_NAME)_darwin_arm64; \
else \
echo "⏭️ Skip darwin/arm64: clang not found"; \
fi; \
fi \
;; \
*) \
echo "❓ Unknown option: $$ch (valid: 1 2 3 4)"; \
;; \
esac; \
done
crossbuild: clean
@echo "Building for multiple platforms..."
@$(MKDIR_BIN)
for platform in $(PLATFORMS); do \
os=$$(echo $$platform | cut -d/ -f1); \
arch=$$(echo $$platform | cut -d/ -f2); \
float=$$(echo $$platform | cut -d/ -f3); \
output_name=bin/chatlog_$${os}_$${arch}; \
[ "$$float" != "" ] && output_name=$$output_name_$$float; \
[ "$$os" = "windows" ] && output_name=$$output_name.exe; \
echo "Building for $$os/$$arch..."; \
echo "Building for $$output_name..."; \
@GOOS=$$os GOARCH=$$arch CGO_ENABLED=1 GOARM=$$float $(GO) build --tags "fts5" -trimpath $(LDFLAGS) -o $$output_name main.go ; \
if [ "$(ENABLE_UPX)" = "1" ] && echo "$(UPX_PLATFORMS)" | grep -q "$$os/$$arch"; then \
echo "Compressing binary $$output_name..." && upx --best $$output_name; \
fi; \
done
@echo "Done!"
+41 -31
View File
@@ -1,70 +1,80 @@
# Prompt 指南
## 概述
优秀的 `prompt` 可以极大的提高 `chatlog` 使用体验,收集了部分群友分享的 `prompt`,供大家参考。
在处理聊天记录时,尽量选择上下文长度足够的 LLM,例如 `Gemini 2.5 Pro`、`Claude 3.5 Sonnet` 等。
优秀的 `prompt` 可以极大的提高 `chatlog` 使用体验,收集了部分群友分享的 `prompt`,供大家参考。
在处理聊天记录时,尽量选择上下文长度足够的 LLM,例如 `Gemini 2.5 Pro`、`Claude 3.5 Sonnet` 等。
欢迎大家在 [Discussions](https://github.com/sjzar/chatlog/discussions/47) 中分享自己的使用方式,共同进步。
## 群聊总结
作者:@eyaeya
```md
你是一个中文的群聊总结的助手,你可以为一个微信的群聊记录,提取并总结每个时间段大家在重点讨论的话题内容。
请帮我将 "<talker>" 在 <Time> 的群聊内容总结成一个群聊报告,包含不多于5个的话题的总结(如果还有更多话题,可以在后面简单补充)。每个话题包含以下内容:
- 话题名(50字以内,带序号1️⃣2️⃣3️⃣,同时附带热度,以🔥数量表示)
- 参与者(不超过5个人,将重复的人名去重)
- 时间段(从几点到几点)
- 过程(50到200字左右)
- 评价(50字以下)
- 分割线: ------------
请帮我将 "<talker>" 在 <Time> 的群聊内容总结成一个群聊报告,包含不多于 5 个的话题的总结(如果还有更多话题,可以在后面简单补充)。每个话题包含以下内容:
- 话题名(50 字以内,带序号 1️⃣2️⃣3️⃣,同时附带热度,以 🔥 数量表示)
- 参与者(不超过 5 个人,将重复的人名去重)
- 时间段(从几点到几点)
- 过程(50 到 200 字左右)
- 评价(50 字以下)
- 分割线: ------------
另外有以下要求:
1. 每个话题结束使用 ------------ 分割
2. 使用中文冒号
3. 无需大标题
4. 开始给出本群讨论风格的整体评价,例如活跃、太水、太黄、太暴力、话题不集中、无聊诸如此类
最后总结下最活跃的前五个发言者。
最后总结下最活跃的前五个发言者。
```
## 微信聊天记录可视化
作者:@数字声明卡兹克
原文地址:https://mp.weixin.qq.com/s/Z66YRjY1EnC_hMgXE9_nnw
Prompt:[微信聊天记录可视化prompt.txt](https://github.com/user-attachments/files/19773263/prompt.txt)
作者:@数字声明卡兹克
[原文地址](https://mp.weixin.qq.com/s/Z66YRjY1EnC_hMgXE9_nnw)
Prompt:[微信聊天记录可视化 prompt.txt](https://github.com/user-attachments/files/19773263/prompt.txt)
这份 prompt 可以使用聊天记录生成 HTML 网页,再使用 [YOURWARE](https://www.yourware.so/) 部署为可分享的静态网页。
### 技术讨论分析
作者:@eyaeya
```md
你作为一个专业的技术讨论分析者,请对以下聊天记录进行分析和结构化总结:
1. 基础信息提取:
- 将每个主题分成独立的问答对
- 保持原始对话的时间顺序
- 将每个主题分成独立的问答对
- 保持原始对话的时间顺序
1. 问题分析要点:
- 提取问题的具体场景和背景
- 识别问题的核心技术难点
- 突出问题的实际影响
- 提取问题的具体场景和背景
- 识别问题的核心技术难点
- 突出问题的实际影响
1. 解决方案总结:
- 列出具体的解决步骤
- 提取关键工具和资源
- 包含实践经验和注意事项
- 保留重要的链接和参考资料
- 列出具体的解决步骤
- 提取关键工具和资源
- 包含实践经验和注意事项
- 保留重要的链接和参考资料
1. 输出格式:
- 不要输出"日期:YYYY-MM-DD"这一行,直接从问题1开始
- 问题1:<简明扼要的问题描述>
- 回答1:<完整的解决方案>
- 补充:<额外的讨论要点或注意事项>
- 不要输出"日期:YYYY-MM-DD"这一行,直接从问题 1 开始
- 问题 1:<简明扼要的问题描述>
- 回答 1:<完整的解决方案>
- 补充:<额外的讨论要点或注意事项>
1. 额外要求(严格执行):
- 如果有多个相关问题,保持逻辑顺序
- 标记重要的警告和建议、突出经验性的分享内容、保留有价值的专业术语解释、移除"我来分析"等过渡语确保链接的完整性
- 直接以日期开始,不要添加任何开场白
```
- 如果有多个相关问题,保持逻辑顺序
- 标记重要的警告和建议、突出经验性的分享内容、保留有价值的专业术语解释、移除"我来分析"等过渡语确保链接的完整性
- 直接以日期开始,不要添加任何开场白
```
+8
View File
@@ -8,6 +8,7 @@ import (
"github.com/sjzar/chatlog/internal/chatlog/conf"
"github.com/sjzar/chatlog/internal/chatlog/webhook"
"github.com/sjzar/chatlog/internal/errors"
"github.com/sjzar/chatlog/internal/model"
"github.com/sjzar/chatlog/internal/wechatdb"
)
@@ -100,6 +101,13 @@ func (s *Service) GetMessages(start, end time.Time, talker string, sender string
return s.db.GetMessages(start, end, talker, sender, keyword, limit, offset)
}
func (s *Service) SearchMessages(req *model.SearchRequest) (*model.SearchResponse, error) {
if s.db == nil {
return nil, errors.InvalidArg("search before db ready")
}
return s.db.SearchMessages(req)
}
func (s *Service) GetContacts(key string, limit, offset int) (*wechatdb.GetContactsResp, error) {
return s.db.GetContacts(key, limit, offset)
}
+110 -1
View File
@@ -76,6 +76,7 @@ func (s *Service) initAPIRouter() {
api.GET("/session", s.handleSessions)
api.GET("/diary", s.handleDiary)
api.GET("/dashboard", s.handleDashboard)
api.GET("/search", s.handleSearch)
}
}
@@ -85,7 +86,6 @@ func (s *Service) initMCPRouter() {
s.router.Any("/message", func(c *gin.Context) { s.mcpSSEServer.ServeHTTP(c.Writer, c.Request) })
}
// handleDashboard 返回真实统计数据的仪表盘 JSON
// GET /api/v1/dashboard
func (s *Service) handleDashboard(c *gin.Context) {
// 基础聚合
@@ -712,6 +712,115 @@ func estimateDBSize(workDir string) int64 {
return total
}
func (s *Service) handleSearch(c *gin.Context) {
params := struct {
Query string `form:"q"`
Talker string `form:"talker"`
Sender string `form:"sender"`
Time string `form:"time"`
Start string `form:"start"`
End string `form:"end"`
Limit int `form:"limit"`
Offset int `form:"offset"`
}{}
if err := c.BindQuery(&params); err != nil {
errors.Err(c, err)
return
}
query := strings.TrimSpace(params.Query)
if query == "" {
errors.Err(c, errors.InvalidArg("q"))
return
}
talker := strings.TrimSpace(params.Talker)
if talker == "" {
errors.Err(c, errors.ErrTalkerEmpty)
return
}
limit := params.Limit
if limit <= 0 {
limit = 20
}
if limit > 200 {
limit = 200
}
offset := params.Offset
if offset < 0 {
offset = 0
}
req := &model.SearchRequest{
Query: query,
Talker: talker,
Sender: strings.TrimSpace(params.Sender),
Limit: limit,
Offset: offset,
}
if params.Time != "" {
start, end, ok := util.TimeRangeOf(params.Time)
if !ok {
errors.Err(c, errors.InvalidArg("time"))
return
}
req.Start = start
req.End = end
} else {
if params.Start != "" && params.End != "" {
start, end, ok := util.TimeRangeOf(params.Start + "~" + params.End)
if !ok {
errors.Err(c, errors.InvalidArg("time"))
return
}
req.Start = start
req.End = end
} else if params.Start != "" {
start, end, ok := util.TimeRangeOf(params.Start)
if !ok {
errors.Err(c, errors.InvalidArg("start"))
return
}
req.Start = start
req.End = end
} else if params.End != "" {
start, end, ok := util.TimeRangeOf(params.End)
if !ok {
errors.Err(c, errors.InvalidArg("end"))
return
}
req.Start = start
req.End = end
}
}
if !req.Start.IsZero() && !req.End.IsZero() && req.End.Before(req.Start) {
req.Start, req.End = req.End, req.Start
}
resp, err := s.db.SearchMessages(req)
if err != nil {
errors.Err(c, err)
return
}
if resp == nil {
resp = &model.SearchResponse{Hits: []*model.SearchHit{}, Limit: limit, Offset: offset}
}
resp.Query = req.Query
resp.Talker = req.Talker
resp.Sender = req.Sender
resp.Start = req.Start
resp.End = req.End
resp.Limit = limit
resp.Offset = offset
c.JSON(http.StatusOK, resp)
}
func (s *Service) handleChatlog(c *gin.Context) {
q := struct {
Time string `form:"time"`
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -21,7 +21,7 @@ func (e *Error) Error() string {
if e.Cause != nil {
return fmt.Sprintf("%s: %v", e.Message, e.Cause)
}
return fmt.Sprintf("%s", e.Message)
return e.Message
}
func (e *Error) String() string {
+4
View File
@@ -65,3 +65,7 @@ func InitCacheFailed(cause error) *Error {
func FileGroupNotFound(name string) *Error {
return Newf(nil, http.StatusNotFound, "file group not found: %s", name).WithStack()
}
func SearchNotSupported(platform string, version int) *Error {
return Newf(nil, http.StatusNotImplemented, "search not supported for %s v%d", platform, version).WithStack()
}
+52
View File
@@ -0,0 +1,52 @@
package model
import "time"
// SearchRequest 表示一次搜索查询的参数
// Start 和 End 为闭区间;如未提供则由调用方决定默认范围
// Limit/Offset 由调用链路在进入数据源前进行裁剪
// Sender 使用英文逗号分隔多个筛选条件
// Talker 目前要求调用方传入具体会话(可多选,英文逗号分隔)
type SearchRequest struct {
Query string `json:"query"`
Talker string `json:"talker"`
Sender string `json:"sender"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// Clone 生成请求的浅拷贝,便于在不同层级添加额外参数
func (r *SearchRequest) Clone() *SearchRequest {
if r == nil {
return nil
}
copy := *r
return &copy
}
// SearchHit 表示一次搜索命中的消息及其高亮片段
// Score 使用 SQLite FTS5 的 bm25 分值,越小代表相关度越高
type SearchHit struct {
Message *Message `json:"message"`
Snippet string `json:"snippet"`
Score float64 `json:"score"`
}
// SearchResponse 汇总搜索结果
// DurationMs 统计搜索耗时(毫秒),仅供参考
// Limit / Offset 为实际生效的分页参数
// Hits 序列按相关度排序,命中数可能小于 limit(例如过滤后不足)
type SearchResponse struct {
Total int `json:"total"`
Hits []*SearchHit `json:"hits"`
DurationMs int64 `json:"duration_ms"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Query string `json:"query"`
Talker string `json:"talker"`
Sender string `json:"sender"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
}
@@ -511,6 +511,10 @@ func (ds *DataSource) GetMessages(ctx context.Context, startTime, endTime time.T
return filteredMessages, nil
}
func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchRequest) (*model.SearchResponse, error) {
return nil, errors.SearchNotSupported("darwin", 3)
}
// 从表名中提取 talker
func extractTalkerFromTableName(tableName string) string {
@@ -17,6 +17,7 @@ type DataSource interface {
// 消息
GetMessages(ctx context.Context, startTime, endTime time.Time, talker string, sender string, keyword string, limit, offset int) ([]*model.Message, error)
SearchMessages(ctx context.Context, req *model.SearchRequest) (*model.SearchResponse, error)
// 联系人
GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error)
@@ -6,6 +6,7 @@ import (
"database/sql"
"encoding/hex"
"fmt"
"math"
"regexp"
"sort"
"strings"
@@ -368,6 +369,281 @@ func (ds *DataSource) GetMessages(ctx context.Context, startTime, endTime time.T
return filteredMessages, nil
}
func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchRequest) (*model.SearchResponse, error) {
if req == nil {
return nil, errors.InvalidArg("request")
}
ftsQuery := util.BuildFTSQuery(req.Query)
if ftsQuery == "" {
return &model.SearchResponse{
Total: 0,
Hits: []*model.SearchHit{},
DurationMs: 0,
Limit: req.Limit,
Offset: req.Offset,
Query: req.Query,
Talker: req.Talker,
Sender: req.Sender,
Start: req.Start,
End: req.End,
}, nil
}
talkers := util.Str2List(req.Talker, ",")
if len(talkers) == 0 {
return nil, errors.ErrTalkerEmpty
}
limit := req.Limit
if limit <= 0 {
limit = 20
}
if limit > 200 {
limit = 200
}
offset := req.Offset
if offset < 0 {
offset = 0
}
req.Limit = limit
req.Offset = offset
budget := limit + offset + 20
if budget < 50 {
budget = 50
}
if budget > 1000 {
budget = 1000
}
startRange := req.Start
endRange := req.End
if startRange.IsZero() {
startRange = time.Unix(0, 0)
}
if endRange.IsZero() {
endRange = time.Now().Add(24 * time.Hour)
}
if endRange.Before(startRange) {
startRange, endRange = endRange, startRange
}
startUnix := int64(0)
if !req.Start.IsZero() {
startUnix = req.Start.Unix()
}
endUnix := int64(math.MaxInt64)
if !req.End.IsZero() {
endUnix = req.End.Unix()
}
if endUnix < startUnix {
startUnix, endUnix = endUnix, startUnix
}
senders := util.Str2List(req.Sender, ",")
senderFilter := len(senders) > 0
senderSet := make(map[string]struct{}, len(senders))
for _, s := range senders {
senderSet[s] = struct{}{}
}
startTs := time.Now()
allHits := make([]*model.SearchHit, 0, limit+offset)
seenSeq := make(map[int64]struct{})
dbInfos := ds.getDBInfosForTimeRange(startRange, endRange)
if len(dbInfos) == 0 {
return &model.SearchResponse{
Total: 0,
Hits: []*model.SearchHit{},
DurationMs: time.Since(startTs).Milliseconds(),
Limit: limit,
Offset: offset,
Query: req.Query,
Talker: req.Talker,
Sender: req.Sender,
Start: req.Start,
End: req.End,
}, nil
}
for _, dbInfo := range dbInfos {
if err := ctx.Err(); err != nil {
return nil, err
}
db, err := ds.dbm.OpenDB(dbInfo.FilePath)
if err != nil {
log.Err(err).Msgf("数据库 %s 未打开", dbInfo.FilePath)
continue
}
for _, talker := range talkers {
if err := ctx.Err(); err != nil {
return nil, err
}
err := func() error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return errors.QueryFailed("begin tx", err)
}
defer func() {
_ = tx.Rollback()
}()
talkerHash := md5.Sum([]byte(talker))
tableName := "Msg_" + hex.EncodeToString(talkerHash[:])
var exists int
if err := tx.QueryRowContext(ctx, `SELECT 1 FROM sqlite_master WHERE type='table' AND name=?`, tableName).Scan(&exists); err != nil {
if err == sql.ErrNoRows {
return nil
}
return errors.QueryFailed("check table", err)
}
if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS chatlog_msg_base(seq INTEGER PRIMARY KEY, content TEXT)`); err != nil {
return errors.QueryFailed("create temp table", err)
}
if _, err := tx.ExecContext(ctx, `DELETE FROM chatlog_msg_base`); err != nil {
return errors.QueryFailed("cleanup temp table", err)
}
insertSQL := fmt.Sprintf(`INSERT INTO chatlog_msg_base(seq, content)
SELECT sort_seq, CAST(message_content AS TEXT)
FROM %s
WHERE create_time >= ? AND create_time <= ?
AND message_content IS NOT NULL AND LENGTH(message_content) > 0`, tableName)
if _, err := tx.ExecContext(ctx, insertSQL, startUnix, endUnix); err != nil {
return errors.QueryFailed("insert temp base", err)
}
if _, err := tx.ExecContext(ctx, `CREATE VIRTUAL TABLE IF NOT EXISTS temp.chatlog_msg_fts USING fts5(content, tokenize='unicode61')`); err != nil {
return errors.QueryFailed("create fts", err)
}
if _, err := tx.ExecContext(ctx, `DELETE FROM temp.chatlog_msg_fts`); err != nil {
return errors.QueryFailed("cleanup fts", err)
}
if _, err := tx.ExecContext(ctx, `INSERT INTO temp.chatlog_msg_fts(rowid, content) SELECT seq, content FROM chatlog_msg_base`); err != nil {
return errors.QueryFailed("insert fts", err)
}
querySQL := fmt.Sprintf(`
SELECT m.sort_seq, m.server_id, m.local_type, n.user_name,
m.create_time, m.message_content, m.packed_info_data, m.status,
highlight(chatlog_msg_fts, 0, '<mark>', '</mark>') AS snippet,
bm25(chatlog_msg_fts) AS score
FROM temp.chatlog_msg_fts
JOIN %s AS m ON m.sort_seq = chatlog_msg_fts.rowid
LEFT JOIN Name2Id n ON m.real_sender_id = n.rowid
WHERE chatlog_msg_fts MATCH ?
ORDER BY score
LIMIT ?
`, tableName)
rows, err := tx.QueryContext(ctx, querySQL, ftsQuery, budget)
if err != nil {
return errors.QueryFailed("fts query", err)
}
defer rows.Close()
for rows.Next() {
var msg model.MessageV4
var messageContent []byte
var snippet sql.NullString
var score sql.NullFloat64
if err := rows.Scan(&msg.SortSeq, &msg.ServerID, &msg.LocalType, &msg.UserName,
&msg.CreateTime, &messageContent, &msg.PackedInfoData, &msg.Status, &snippet, &score); err != nil {
return errors.ScanRowFailed(err)
}
msg.MessageContent = messageContent
message := msg.Wrap(talker)
if senderFilter {
if _, ok := senderSet[message.Sender]; !ok {
continue
}
}
if _, dup := seenSeq[message.Seq]; dup {
continue
}
seenSeq[message.Seq] = struct{}{}
hitSnippet := ""
if snippet.Valid {
hitSnippet = snippet.String
}
hitScore := 0.0
if score.Valid {
hitScore = score.Float64
}
allHits = append(allHits, &model.SearchHit{Message: message, Snippet: hitSnippet, Score: hitScore})
}
if err := rows.Err(); err != nil {
return errors.QueryFailed("fts rows", err)
}
if err := tx.Commit(); err != nil {
return errors.QueryFailed("commit tx", err)
}
return nil
}()
if err != nil {
return nil, err
}
}
}
if len(allHits) == 0 {
return &model.SearchResponse{
Total: 0,
Hits: []*model.SearchHit{},
DurationMs: time.Since(startTs).Milliseconds(),
Limit: limit,
Offset: offset,
Query: req.Query,
Talker: req.Talker,
Sender: req.Sender,
Start: req.Start,
End: req.End,
}, nil
}
sort.SliceStable(allHits, func(i, j int) bool {
if allHits[i].Score == allHits[j].Score {
return allHits[i].Message.Seq < allHits[j].Message.Seq
}
return allHits[i].Score < allHits[j].Score
})
total := len(allHits)
if offset > total {
offset = total
}
endIndex := offset + limit
if endIndex > total {
endIndex = total
}
paged := allHits[offset:endIndex]
return &model.SearchResponse{
Total: total,
Hits: paged,
DurationMs: time.Since(startTs).Milliseconds(),
Limit: limit,
Offset: offset,
Query: req.Query,
Talker: req.Talker,
Sender: req.Sender,
Start: req.Start,
End: req.End,
}, nil
}
// 联系人
func (ds *DataSource) GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error) {
var query string
@@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/hex"
"fmt"
"math"
"regexp"
"sort"
"strings"
@@ -405,6 +406,276 @@ func (ds *DataSource) GetMessages(ctx context.Context, startTime, endTime time.T
return filteredMessages, nil
}
func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchRequest) (*model.SearchResponse, error) {
if req == nil {
return nil, errors.InvalidArg("request")
}
ftsQuery := util.BuildFTSQuery(req.Query)
if ftsQuery == "" {
return &model.SearchResponse{
Total: 0,
Hits: []*model.SearchHit{},
DurationMs: 0,
Limit: req.Limit,
Offset: req.Offset,
Query: req.Query,
Talker: req.Talker,
Sender: req.Sender,
Start: req.Start,
End: req.End,
}, nil
}
talkers := util.Str2List(req.Talker, ",")
if len(talkers) == 0 {
return nil, errors.ErrTalkerEmpty
}
limit := req.Limit
if limit <= 0 {
limit = 20
}
if limit > 200 {
limit = 200
}
offset := req.Offset
if offset < 0 {
offset = 0
}
req.Limit = limit
req.Offset = offset
budget := limit + offset + 20
if budget < 50 {
budget = 50
}
if budget > 1000 {
budget = 1000
}
startRange := req.Start
endRange := req.End
if startRange.IsZero() {
startRange = time.Unix(0, 0)
}
if endRange.IsZero() {
endRange = time.Now().Add(24 * time.Hour)
}
if endRange.Before(startRange) {
startRange, endRange = endRange, startRange
}
startSeq := int64(0)
if !req.Start.IsZero() {
startSeq = req.Start.Unix() * 1000
}
endSeq := int64(math.MaxInt64)
if !req.End.IsZero() {
endSeq = req.End.Unix() * 1000
}
if endSeq < startSeq {
startSeq, endSeq = endSeq, startSeq
}
senders := util.Str2List(req.Sender, ",")
senderFilter := len(senders) > 0
senderSet := make(map[string]struct{}, len(senders))
for _, s := range senders {
senderSet[s] = struct{}{}
}
startTs := time.Now()
allHits := make([]*model.SearchHit, 0, limit+offset)
seenSeq := make(map[int64]struct{})
dbInfos := ds.getDBInfosForTimeRange(startRange, endRange)
if len(dbInfos) == 0 {
return &model.SearchResponse{
Total: 0,
Hits: []*model.SearchHit{},
DurationMs: time.Since(startTs).Milliseconds(),
Limit: limit,
Offset: offset,
Query: req.Query,
Talker: req.Talker,
Sender: req.Sender,
Start: req.Start,
End: req.End,
}, nil
}
for _, dbInfo := range dbInfos {
if err := ctx.Err(); err != nil {
return nil, err
}
db, err := ds.dbm.OpenDB(dbInfo.FilePath)
if err != nil {
log.Err(err).Msgf("数据库 %s 未打开", dbInfo.FilePath)
continue
}
for _, talker := range talkers {
if err := ctx.Err(); err != nil {
return nil, err
}
err := func() error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return errors.QueryFailed("begin tx", err)
}
defer func() {
_ = tx.Rollback()
}()
if _, err := tx.ExecContext(ctx, `CREATE TEMP TABLE IF NOT EXISTS chatlog_msg_base(seq INTEGER PRIMARY KEY, content TEXT)`); err != nil {
return errors.QueryFailed("create temp table", err)
}
if _, err := tx.ExecContext(ctx, `DELETE FROM chatlog_msg_base`); err != nil {
return errors.QueryFailed("cleanup temp table", err)
}
conditions := []string{"Sequence >= ?", "Sequence <= ?", "StrContent IS NOT NULL", "LENGTH(StrContent) > 0"}
args := []interface{}{startSeq, endSeq}
if talkerID, ok := dbInfo.TalkerMap[talker]; ok {
conditions = append(conditions, "TalkerId = ?")
args = append(args, talkerID)
} else {
conditions = append(conditions, "StrTalker = ?")
args = append(args, talker)
}
insertSQL := fmt.Sprintf(`INSERT INTO chatlog_msg_base(seq, content)
SELECT Sequence, StrContent FROM MSG WHERE %s`, strings.Join(conditions, " AND "))
if _, err := tx.ExecContext(ctx, insertSQL, args...); err != nil {
return errors.QueryFailed("insert temp base", err)
}
if _, err := tx.ExecContext(ctx, `CREATE VIRTUAL TABLE IF NOT EXISTS temp.chatlog_msg_fts USING fts5(content, tokenize='unicode61')`); err != nil {
return errors.QueryFailed("create fts", err)
}
if _, err := tx.ExecContext(ctx, `DELETE FROM temp.chatlog_msg_fts`); err != nil {
return errors.QueryFailed("cleanup fts", err)
}
if _, err := tx.ExecContext(ctx, `INSERT INTO temp.chatlog_msg_fts(rowid, content) SELECT seq, content FROM chatlog_msg_base`); err != nil {
return errors.QueryFailed("insert fts", err)
}
rows, err := tx.QueryContext(ctx, `
SELECT m.MsgSvrID, m.Sequence, m.CreateTime, m.StrTalker, m.IsSender,
m.Type, m.SubType, m.StrContent, m.CompressContent, m.BytesExtra,
highlight(chatlog_msg_fts, 0, '<mark>', '</mark>') AS snippet,
bm25(chatlog_msg_fts) AS score
FROM temp.chatlog_msg_fts
JOIN MSG m ON m.Sequence = chatlog_msg_fts.rowid
WHERE chatlog_msg_fts MATCH ?
ORDER BY score
LIMIT ?
`, ftsQuery, budget)
if err != nil {
return errors.QueryFailed("fts query", err)
}
defer rows.Close()
for rows.Next() {
var msg model.MessageV3
var compressContent []byte
var bytesExtra []byte
var snippet sql.NullString
var score sql.NullFloat64
if err := rows.Scan(&msg.MsgSvrID, &msg.Sequence, &msg.CreateTime, &msg.StrTalker, &msg.IsSender,
&msg.Type, &msg.SubType, &msg.StrContent, &compressContent, &bytesExtra, &snippet, &score); err != nil {
return errors.ScanRowFailed(err)
}
msg.CompressContent = compressContent
msg.BytesExtra = bytesExtra
message := msg.Wrap()
if senderFilter {
if _, ok := senderSet[message.Sender]; !ok {
continue
}
}
if _, dup := seenSeq[message.Seq]; dup {
continue
}
seenSeq[message.Seq] = struct{}{}
hitSnippet := ""
if snippet.Valid {
hitSnippet = snippet.String
}
hitScore := 0.0
if score.Valid {
hitScore = score.Float64
}
allHits = append(allHits, &model.SearchHit{Message: message, Snippet: hitSnippet, Score: hitScore})
}
if err := rows.Err(); err != nil {
return errors.QueryFailed("fts rows", err)
}
if err := tx.Commit(); err != nil {
return errors.QueryFailed("commit tx", err)
}
return nil
}()
if err != nil {
return nil, err
}
}
}
if len(allHits) == 0 {
return &model.SearchResponse{
Total: 0,
Hits: []*model.SearchHit{},
DurationMs: time.Since(startTs).Milliseconds(),
Limit: limit,
Offset: offset,
Query: req.Query,
Talker: req.Talker,
Sender: req.Sender,
Start: req.Start,
End: req.End,
}, nil
}
sort.SliceStable(allHits, func(i, j int) bool {
if allHits[i].Score == allHits[j].Score {
return allHits[i].Message.Seq < allHits[j].Message.Seq
}
return allHits[i].Score < allHits[j].Score
})
total := len(allHits)
if offset > total {
offset = total
}
endIndex := offset + limit
if endIndex > total {
endIndex = total
}
paged := allHits[offset:endIndex]
return &model.SearchResponse{
Total: total,
Hits: paged,
DurationMs: time.Since(startTs).Milliseconds(),
Limit: limit,
Offset: offset,
Query: req.Query,
Talker: req.Talker,
Sender: req.Sender,
Start: req.Start,
End: req.End,
}, nil
}
// GetContacts 实现获取联系人信息的方法
func (ds *DataSource) GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error) {
var query string
+52
View File
@@ -0,0 +1,52 @@
package repository
import (
"context"
"github.com/rs/zerolog/log"
"github.com/sjzar/chatlog/internal/errors"
"github.com/sjzar/chatlog/internal/model"
)
// SearchMessages 执行全文检索,并在返回前补充联系人/群聊信息。
func (r *Repository) SearchMessages(ctx context.Context, req *model.SearchRequest) (*model.SearchResponse, error) {
if req == nil {
return nil, errors.InvalidArg("request")
}
nReq := req.Clone()
if nReq == nil {
nReq = &model.SearchRequest{}
}
// 兼容现有的联系人/群聊别名:在进入数据源前将 talker/sender 解析成真实 userName
normalizedTalker, normalizedSender := r.parseTalkerAndSender(ctx, nReq.Talker, nReq.Sender)
nReq.Talker = normalizedTalker
nReq.Sender = normalizedSender
resp, err := r.ds.SearchMessages(ctx, nReq)
if err != nil {
return nil, err
}
if resp == nil {
return &model.SearchResponse{Hits: []*model.SearchHit{}, Limit: nReq.Limit, Offset: nReq.Offset}, nil
}
// Enrich message metadata(头像、群昵称、显示名等)
messages := make([]*model.Message, 0, len(resp.Hits))
for _, hit := range resp.Hits {
if hit == nil || hit.Message == nil {
continue
}
messages = append(messages, hit.Message)
}
if len(messages) > 0 {
if err := r.EnrichMessages(ctx, messages); err != nil {
log.Debug().Msgf("EnrichMessages in search failed: %v", err)
}
}
return resp, nil
}
+5
View File
@@ -70,6 +70,11 @@ func (w *DB) GetMessages(start, end time.Time, talker string, sender string, key
return messages, nil
}
func (w *DB) SearchMessages(req *model.SearchRequest) (*model.SearchResponse, error) {
ctx := context.Background()
return w.repo.SearchMessages(ctx, req)
}
type GetContactsResp struct {
Items []*model.Contact `json:"items"`
}
-3
View File
@@ -1,6 +1,3 @@
//go:build cgo
// +build cgo
package silk
import (
-11
View File
@@ -1,11 +0,0 @@
//go:build !cgo
// +build !cgo
package silk
import "fmt"
// Silk2MP3 is a no-op stub when built without CGO; returns an error so callers can fall back.
func Silk2MP3(data []byte) ([]byte, error) {
return nil, fmt.Errorf("silk2mp3 unavailable: built without cgo")
}
+42
View File
@@ -69,3 +69,45 @@ func Str2List(str string, sep string) []string {
return list
}
// BuildFTSQuery 将用户输入转换为 SQLite FTS5 可以安全解析的查询表达式。
// - 如果输入包含 AND/OR/NEAR/()/"/* 等高级语法,则直接返回原样,允许高级用户自行控制;
// - 否则将空白分隔的关键词转换为 "term" AND "term2" 的形式,避免意外的模糊匹配;
// - 同时会对双引号做转义处理,避免 MATCH 解析失败。
func BuildFTSQuery(input string) string {
s := strings.TrimSpace(input)
if s == "" {
return ""
}
upper := strings.ToUpper(s)
advanced := strings.ContainsAny(s, "\"'*()") ||
strings.Contains(upper, " AND ") ||
strings.Contains(upper, " OR ") ||
strings.Contains(upper, " NEAR ") ||
strings.HasPrefix(upper, "NOT ")
if advanced {
return s
}
tokens := strings.Fields(s)
if len(tokens) == 0 {
return ""
}
parts := make([]string, 0, len(tokens))
for _, token := range tokens {
t := strings.TrimSpace(token)
if t == "" {
continue
}
t = strings.ReplaceAll(t, "\"", "\"\"")
parts = append(parts, fmt.Sprintf("\"%s\"", t))
}
if len(parts) == 0 {
return ""
}
return strings.Join(parts, " AND ")
}