From 687ee63aa6c3fae7b77e403d404455835518de35 Mon Sep 17 00:00:00 2001 From: Symb0x76 <9667434@qq.com> Date: Thu, 9 Oct 2025 20:09:07 +0800 Subject: [PATCH] !refactor(search): Use Bleve in replace of SQLite FTS5 --- go.mod | 25 + go.sum | 53 ++ internal/chatlog/http/route.go | 246 ++++++++- internal/chatlog/http/static/index.htm | 380 ++++++++++---- internal/chatlog/webhook/webhook.go | 3 +- internal/model/search.go | 33 +- .../datasource/darwinv3/datasource.go | 4 + internal/wechatdb/datasource/datasource.go | 1 + .../wechatdb/datasource/dbm/fingerprint.go | 83 +++ internal/wechatdb/datasource/v4/datasource.go | 4 + .../datasource/windowsv3/datasource.go | 150 +++++- internal/wechatdb/indexer/indexer.go | 482 ++++++++++++++++++ internal/wechatdb/repository/repository.go | 4 +- 13 files changed, 1350 insertions(+), 118 deletions(-) create mode 100644 internal/wechatdb/datasource/dbm/fingerprint.go create mode 100644 internal/wechatdb/indexer/indexer.go diff --git a/go.mod b/go.mod index 87074f7..105f540 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.24.0 require ( github.com/Eyevinn/mp4ff v0.49.0 + github.com/blevesearch/bleve/v2 v2.4.2 github.com/cespare/xxhash v1.1.0 github.com/fsnotify/fsnotify v1.9.0 github.com/gdamore/tcell/v2 v2.8.1 @@ -29,7 +30,26 @@ require ( ) require ( + github.com/RoaringBitmap/roaring v1.9.3 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/bits-and-blooms/bitset v1.12.0 // indirect + github.com/blevesearch/bleve_index_api v1.1.10 // indirect + github.com/blevesearch/geo v0.1.20 // indirect + github.com/blevesearch/go-faiss v1.0.20 // indirect + github.com/blevesearch/go-porterstemmer v1.0.3 // indirect + github.com/blevesearch/gtreap v0.1.1 // indirect + github.com/blevesearch/mmap-go v1.0.4 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.2.15 // indirect + github.com/blevesearch/segment v0.9.1 // indirect + github.com/blevesearch/snowballstem v0.9.0 // indirect + github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect + github.com/blevesearch/vellum v1.0.10 // indirect + github.com/blevesearch/zapx/v11 v11.3.10 // indirect + github.com/blevesearch/zapx/v12 v12.3.10 // indirect + github.com/blevesearch/zapx/v13 v13.3.10 // indirect + github.com/blevesearch/zapx/v14 v14.3.10 // indirect + github.com/blevesearch/zapx/v15 v15.3.13 // indirect + github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.14.0 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect @@ -44,6 +64,9 @@ require ( github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect + github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect + github.com/golang/protobuf v1.5.0 // indirect + github.com/golang/snappy v0.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -57,6 +80,7 @@ require ( github.com/mattn/go-runewidth v0.0.16 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -73,6 +97,7 @@ require ( github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.20.0 // indirect golang.org/x/net v0.43.0 // indirect golang.org/x/term v0.34.0 // indirect diff --git a/go.sum b/go.sum index 27f66ae..63e157e 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,48 @@ github.com/Eyevinn/mp4ff v0.49.0 h1:00eRg5/KwcLGWUbv+hlifldf74qg46G1IxoWXHrgDvw= github.com/Eyevinn/mp4ff v0.49.0/go.mod h1:hJNUUqOBryLAzUW9wpCJyw2HaI+TCd2rUPhafoS5lgg= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= +github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/blevesearch/bleve/v2 v2.4.2 h1:NooYP1mb3c0StkiY9/xviiq2LGSaE8BQBCc/pirMx0U= +github.com/blevesearch/bleve/v2 v2.4.2/go.mod h1:ATNKj7Yl2oJv/lGuF4kx39bST2dveX6w0th2FFYLkc8= +github.com/blevesearch/bleve_index_api v1.1.10 h1:PDLFhVjrjQWr6jCuU7TwlmByQVCSEURADHdCqVS9+g0= +github.com/blevesearch/bleve_index_api v1.1.10/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= +github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= +github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= +github.com/blevesearch/go-faiss v1.0.20 h1:AIkdTQFWuZ5LQmKQSebgMR4RynGNw8ZseJXaan5kvtI= +github.com/blevesearch/go-faiss v1.0.20/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= +github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= +github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= +github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= +github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= +github.com/blevesearch/scorch_segment_api/v2 v2.2.15 h1:prV17iU/o+A8FiZi9MXmqbagd8I0bCqM7OKUYPbnb5Y= +github.com/blevesearch/scorch_segment_api/v2 v2.2.15/go.mod h1:db0cmP03bPNadXrCDuVkKLV6ywFSiRgPFT1YVrestBc= +github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= +github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= +github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= +github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= +github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= +github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= +github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= +github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= +github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= +github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= +github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= +github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= +github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= +github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= +github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= +github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= +github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi/AUHjU= +github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= @@ -53,6 +93,13 @@ github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= +github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -103,6 +150,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= @@ -176,6 +225,8 @@ github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT0 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -255,6 +306,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/chatlog/http/route.go b/internal/chatlog/http/route.go index b663a54..4d8a78c 100644 --- a/internal/chatlog/http/route.go +++ b/internal/chatlog/http/route.go @@ -722,6 +722,7 @@ func (s *Service) handleSearch(c *gin.Context) { End string `form:"end"` Limit int `form:"limit"` Offset int `form:"offset"` + Format string `form:"format"` }{} if err := c.BindQuery(¶ms); err != nil { @@ -730,16 +731,8 @@ func (s *Service) handleSearch(c *gin.Context) { } query := strings.TrimSpace(params.Query) - if query == "" { - errors.Err(c, errors.InvalidArg("q")) - return - } talker := strings.TrimSpace(params.Talker) - if talker == "" { - errors.Err(c, errors.ErrTalkerEmpty) - return - } limit := params.Limit if limit <= 0 { @@ -818,7 +811,163 @@ func (s *Service) handleSearch(c *gin.Context) { resp.Limit = limit resp.Offset = offset - c.JSON(http.StatusOK, resp) + format := strings.ToLower(strings.TrimSpace(params.Format)) + terms := extractQueryTerms(resp.Query) + + switch format { + case "html": + c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") + c.Writer.WriteString(`Search Result`) + c.Writer.WriteString("
") + c.Writer.WriteString("

搜索结果

") + if resp.Query != "" { + c.Writer.WriteString("

关键词：" + template.HTMLEscapeString(resp.Query) + "

") + } + talkerLabel := "全部会话" + if resp.Talker != "" { + talkerLabel = template.HTMLEscapeString(resp.Talker) + } + c.Writer.WriteString("

会话：" + talkerLabel + "

") + if resp.Sender != "" { + c.Writer.WriteString("

发送者：" + template.HTMLEscapeString(resp.Sender) + "

") + } + timeLabel := "不限" + if !resp.Start.IsZero() && !resp.End.IsZero() { + timeLabel = resp.Start.Format("2006-01-02 15:04:05") + " ~ " + resp.End.Format("2006-01-02 15:04:05") + } else if !resp.Start.IsZero() { + timeLabel = ">= " + resp.Start.Format("2006-01-02 15:04:05") + } else if !resp.End.IsZero() { + timeLabel = "<= " + resp.End.Format("2006-01-02 15:04:05") + } + c.Writer.WriteString("

时间范围：" + template.HTMLEscapeString(timeLabel) + "

") + c.Writer.WriteString(fmt.Sprintf("

命中条数：%d（本页 %d 条）

", resp.Total, len(resp.Hits))) + c.Writer.WriteString("
") + + if len(resp.Hits) == 0 { + c.Writer.WriteString("
暂无搜索结果
") + } else { + c.Writer.WriteString("
") + for idx, hit := range resp.Hits { + if hit == nil || hit.Message == nil { + continue + } + msg := hit.Message + msg.SetContent("host", c.Request.Host) + talkerDisplay := msg.Talker + if msg.TalkerName != "" { + talkerDisplay = fmt.Sprintf("%s (%s)", msg.TalkerName, msg.Talker) + } + senderDisplay := msg.Sender + if msg.IsSelf { + senderDisplay = "我" + } + if msg.SenderName != "" { + senderDisplay = fmt.Sprintf("%s(%s)", msg.SenderName, msg.Sender) + } + c.Writer.WriteString("
") + c.Writer.WriteString("
#" + fmt.Sprintf("%d", idx+1) + " · " + template.HTMLEscapeString(talkerDisplay) + "") + if hit.Score > 0 { + c.Writer.WriteString("score: " + fmt.Sprintf("%.4f", hit.Score) + "") + } + c.Writer.WriteString("
") + c.Writer.WriteString("
时间：" + template.HTMLEscapeString(msg.Time.Format("2006-01-02 15:04:05")) + "发送者：" + template.HTMLEscapeString(senderDisplay) + "
") + if snippet := strings.TrimSpace(hit.Snippet); snippet != "" { + h := highlightPlainTextToHTML(snippet, terms) + h = strings.ReplaceAll(h, "\n", "
") + c.Writer.WriteString("
" + h + "
") + } + c.Writer.WriteString("
" + messageHTMLPlaceholder(msg) + "
") + c.Writer.WriteString("
") + } + c.Writer.WriteString("
") + } + c.Writer.WriteString(previewHTMLSnippet) + c.Writer.WriteString("") + return + case "text", "plain": + c.Writer.Header().Set("Content-Type", "text/plain; charset=utf-8") + c.Writer.Header().Set("Cache-Control", "no-cache") + c.Writer.Header().Set("Connection", "keep-alive") + fmt.Fprintf(c.Writer, "关键词: %s\n", resp.Query) + talkerLabel := resp.Talker + if talkerLabel == "" { + talkerLabel = "全部会话" + } + fmt.Fprintf(c.Writer, "会话: %s\n", talkerLabel) + if resp.Sender != "" { + fmt.Fprintf(c.Writer, "发送者: %s\n", resp.Sender) + } + switch { + case !resp.Start.IsZero() && !resp.End.IsZero(): + fmt.Fprintf(c.Writer, "时间: %s ~ %s\n", resp.Start.Format("2006-01-02 15:04:05"), resp.End.Format("2006-01-02 15:04:05")) + case !resp.Start.IsZero(): + fmt.Fprintf(c.Writer, "时间: >= %s\n", resp.Start.Format("2006-01-02 15:04:05")) + case !resp.End.IsZero(): + fmt.Fprintf(c.Writer, "时间: <= %s\n", resp.End.Format("2006-01-02 15:04:05")) + default: + fmt.Fprintln(c.Writer, "时间: 不限") + } + fmt.Fprintf(c.Writer, "总命中: %d, 本页: %d\n", resp.Total, len(resp.Hits)) + fmt.Fprintln(c.Writer, strings.Repeat("-", 60)) + for idx, hit := range resp.Hits { + if hit == nil || hit.Message == nil { + continue + } + msg := hit.Message + msg.SetContent("host", c.Request.Host) + title := msg.Talker + if msg.TalkerName != "" { + title = fmt.Sprintf("%s (%s)", msg.TalkerName, msg.Talker) + } + sender := msg.Sender + if msg.IsSelf { + sender = "我" + } + if msg.SenderName != "" { + sender = fmt.Sprintf("%s(%s)", msg.SenderName, msg.Sender) + } + fmt.Fprintf(c.Writer, "[%d] %s @ %s\n", idx+1, msg.Time.Format("2006-01-02 15:04:05"), title) + fmt.Fprintf(c.Writer, "发送者: %s\n", sender) + fmt.Fprintf(c.Writer, "%s\n", msg.PlainTextContent()) + if snippet := strings.TrimSpace(hit.Snippet); snippet != "" { + fmt.Fprintf(c.Writer, "Snippet: %s\n", snippet) + } + fmt.Fprintln(c.Writer, strings.Repeat("-", 60)) + } + return + case "csv": + c.Writer.Header().Set("Content-Type", "text/csv; charset=utf-8") + c.Writer.Header().Set("Cache-Control", "no-cache") + c.Writer.Header().Set("Connection", "keep-alive") + c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=search_%s.csv", time.Now().Format("20060102_150405"))) + csvWriter := csv.NewWriter(c.Writer) + csvWriter.Write([]string{"Seq", "Time", "Talker", "TalkerName", "Sender", "SenderName", "Content", "Snippet"}) + for _, hit := range resp.Hits { + if hit == nil || hit.Message == nil { + continue + } + msg := hit.Message + msg.SetContent("host", c.Request.Host) + csvWriter.Write([]string{ + fmt.Sprintf("%d", msg.Seq), + msg.Time.Format("2006-01-02 15:04:05"), + msg.Talker, + msg.TalkerName, + msg.Sender, + msg.SenderName, + msg.PlainTextContent(), + strings.ReplaceAll(hit.Snippet, "\n", " "), + }) + } + csvWriter.Flush() + return + case "json", "": + c.JSON(http.StatusOK, resp) + return + default: + c.JSON(http.StatusOK, resp) + return + } } func (s *Service) handleChatlog(c *gin.Context) { @@ -1501,7 +1650,10 @@ func (s *Service) HandleVoice(c *gin.Context, data []byte) { } // 统一占位符：将 PlainTextContent 里形如 ![标签](url) 或 [标签](url) 的模式转成超链接形式，仅显示 [标签]。 -var placeholderPattern = regexp.MustCompile(`!?\[([^\]]+)\]\((https?://[^)]+)\)`) +var ( + placeholderPattern = regexp.MustCompile(`!?\[([^\]]+)\]\((https?://[^)]+)\)`) + searchTermSplitPattern = regexp.MustCompile(`[\s,，；;、]+`) +) func messageHTMLPlaceholder(m *model.Message) string { content := m.PlainTextContent() @@ -1536,3 +1688,77 @@ func messageHTMLPlaceholder(m *model.Message) string { return `` + anchorText + `` }) } + +func extractQueryTerms(raw string) []string { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + parts := searchTermSplitPattern.Split(raw, -1) + terms := make([]string, 0, len(parts)) + seen := make(map[string]struct{}) + for _, part := range parts { + term := strings.Trim(part, "\"'()[]{}") + if term == "" { + continue + } + switch strings.ToUpper(term) { + case "AND", "OR", "NOT": + continue + } + key := strings.ToLower(term) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + terms = append(terms, term) + } + return terms +} + +func highlightPlainTextToHTML(text string, terms []string) string { + if text == "" { + return "" + } + if len(terms) == 0 { + return template.HTMLEscapeString(text) + } + unique := make(map[string]struct{}) + escaped := make([]string, 0, len(terms)) + for _, term := range terms { + trimmed := strings.TrimSpace(term) + if trimmed == "" { + continue + } + key := strings.ToLower(trimmed) + if _, ok := unique[key]; ok { + continue + } + unique[key] = struct{}{} + escaped = append(escaped, regexp.QuoteMeta(trimmed)) + } + if len(escaped) == 0 { + return template.HTMLEscapeString(text) + } + sort.Slice(escaped, func(i, j int) bool { return len(escaped[i]) > len(escaped[j]) }) + pattern := strings.Join(escaped, "|") + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + return template.HTMLEscapeString(text) + } + matches := re.FindAllStringIndex(text, -1) + if len(matches) == 0 { + return template.HTMLEscapeString(text) + } + var builder strings.Builder + last := 0 + for _, loc := range matches { + builder.WriteString(template.HTMLEscapeString(text[last:loc[0]])) + builder.WriteString(``) + builder.WriteString(template.HTMLEscapeString(text[loc[0]:loc[1]])) + builder.WriteString("") + last = loc[1] + } + builder.WriteString(template.HTMLEscapeString(text[last:])) + return builder.String() +} diff --git a/internal/chatlog/http/static/index.htm b/internal/chatlog/http/static/index.htm index 5d09702..bf406a4 100644 --- a/internal/chatlog/http/static/index.htm +++ b/internal/chatlog/http/static/index.htm @@ -190,6 +190,15 @@ position: relative; } + .result-block { + margin: 0; + white-space: pre-wrap; + font-family: "SFMono-Regular", Consolas, "Liberation Mono", + Menlo, monospace; + font-size: 14px; + line-height: 1.6; + } + .request-url { background-color: #f0f0f0; padding: 10px; @@ -345,6 +354,13 @@ font-style: italic; } + .search-highlight { + background-color: #fff3b0; + color: inherit; + padding: 0 2px; + border-radius: 3px; + } + .required-field { color: var(--error-color); font-weight: bold; @@ -636,24 +652,27 @@ >必填 - +
- 可以从“最近会话”或“群聊”接口获取对应的 ID。 + 可以从“最近会话”或“群聊”接口获取对应的 + ID；如留空将检索所有会话。
@@ -669,18 +688,18 @@ />
- - +
+ + 至 + +
- 留空表示搜索全量消息；支持与聊天记录接口相同的时间表达式。 + 可选择开始日期和结束日期；留空表示不限制时间范围。
@@ -710,6 +729,20 @@ placeholder="默认 0" />
+
+ + +
@@ -833,28 +866,32 @@ errorMessage.style.display = "none"; errorMessage.textContent = ""; + let highlightTerms = []; + let responseFormat = ""; + try { - // 获取当前活动的标签 const activeTab = document .querySelector(".tab.active") .getAttribute("data-tab"); let url = "/api/v1/"; - let params = new URLSearchParams(); + const params = new URLSearchParams(); - // 根据不同的标签构建不同的请求 switch (activeTab) { - case "chatlog": + case "chatlog": { url += "chatlog"; const startDate = document.getElementById("start-date").value; const endDate = document.getElementById("end-date").value; - const talker = - document.getElementById("talker").value; - const sender = - document.getElementById("sender").value; - const keyword = - document.getElementById("keyword").value; + const talker = document + .getElementById("talker") + .value.trim(); + const sender = document + .getElementById("sender") + .value.trim(); + const keyword = document + .getElementById("keyword") + .value.trim(); const limit = document.getElementById("limit").value; const offset = @@ -862,7 +899,6 @@ const format = document.getElementById("format").value; - // 验证必填项：开始日期 if (!startDate && !endDate) { errorMessage.textContent = "错误: 请至少选择开始日期！"; @@ -878,7 +914,6 @@ } let timeValue = ""; - if (startDate && endDate) { if ( new Date(startDate) > new Date(endDate) @@ -901,70 +936,85 @@ } params.append("time", timeValue); - // talker 可为空，留空表示全部按会话分组 if (talker) params.append("talker", talker); if (sender) params.append("sender", sender); - if (keyword) params.append("keyword", keyword); + if (keyword) { + params.append("keyword", keyword); + highlightTerms = + extractSearchTerms(keyword); + } if (limit) params.append("limit", limit); if (offset) params.append("offset", offset); - if (format) params.append("format", format); + if (format) { + params.append("format", format); + responseFormat = format; + } break; - - case "contact": + } + case "contact": { url += "contact"; - const contactKeyword = - document.getElementById( - "contact-keyword" - ).value; + const contactKeyword = document + .getElementById("contact-keyword") + .value.trim(); const contactFormat = document.getElementById( "contact-format" ).value; - if (contactKeyword) + if (contactKeyword) { params.append("keyword", contactKeyword); - if (contactFormat) + highlightTerms = + extractSearchTerms(contactKeyword); + } + if (contactFormat) { params.append("format", contactFormat); + responseFormat = contactFormat; + } break; - - case "chatroom": + } + case "chatroom": { url += "chatroom"; - const chatroomKeyword = - document.getElementById( - "chatroom-keyword" - ).value; + const chatroomKeyword = document + .getElementById("chatroom-keyword") + .value.trim(); const chatroomFormat = document.getElementById( "chatroom-format" ).value; - if (chatroomKeyword) + if (chatroomKeyword) { params.append("keyword", chatroomKeyword); - if (chatroomFormat) + highlightTerms = + extractSearchTerms(chatroomKeyword); + } + if (chatroomFormat) { params.append("format", chatroomFormat); + responseFormat = chatroomFormat; + } break; - - case "session": + } + case "session": { url += "session"; const sessionFormat = document.getElementById( "session-format" ).value; - - if (sessionFormat) + if (sessionFormat) { params.append("format", sessionFormat); + responseFormat = sessionFormat; + } break; - case "diary": + } + case "diary": { url += "diary"; const diaryDateInput = document.getElementById("diary-date"); const diaryDate = diaryDateInput ? diaryDateInput.value : ""; - const diaryTalker = - document.getElementById( - "diary-talker" - ).value; + const diaryTalker = document + .getElementById("diary-talker") + .value.trim(); const diaryFormat = document.getElementById( "diary-format" @@ -978,12 +1028,18 @@ } params.append("date", diaryDate); - if (diaryTalker) + if (diaryTalker) { params.append("talker", diaryTalker); - if (diaryFormat) + highlightTerms = + extractSearchTerms(diaryTalker); + } + if (diaryFormat) { params.append("format", diaryFormat); + responseFormat = diaryFormat; + } break; - case "search": + } + case "search": { url += "search"; const searchQuery = document.getElementById( @@ -997,9 +1053,13 @@ document.getElementById( "search-sender" ).value; - const searchTime = + const searchStartDate = document.getElementById( - "search-time" + "search-start-date" + ).value; + const searchEndDate = + document.getElementById( + "search-end-date" ).value; const searchLimit = document.getElementById( @@ -1009,6 +1069,11 @@ document.getElementById( "search-offset" ).value; + const searchFormatSelect = + document.getElementById("search-format"); + const searchFormatValue = searchFormatSelect + ? searchFormatSelect.value + : ""; if (!searchQuery.trim()) { errorMessage.textContent = @@ -1016,74 +1081,101 @@ errorMessage.style.display = "block"; return; } - if (!searchTalker.trim()) { + + const trimmedQuery = searchQuery.trim(); + highlightTerms = + extractSearchTerms(trimmedQuery); + params.append("q", trimmedQuery); + + const trimmedTalker = searchTalker.trim(); + if (trimmedTalker) { + params.append("talker", trimmedTalker); + } + + let searchTimeValue = ""; + if (searchStartDate && searchEndDate) { + if ( + new Date(searchStartDate) > + new Date(searchEndDate) + ) { + errorMessage.textContent = + "错误: 结束日期不能早于开始日期！"; + errorMessage.style.display = "block"; + return; + } + searchTimeValue = `${searchStartDate}~${searchEndDate}`; + } else if (searchStartDate) { + searchTimeValue = searchStartDate; + } else if (!searchStartDate && searchEndDate) { errorMessage.textContent = - "错误: 请输入会话 ID！"; + "错误: 请先选择开始日期！"; errorMessage.style.display = "block"; return; } - params.append("q", searchQuery.trim()); - params.append("talker", searchTalker.trim()); if (searchSender) params.append( "sender", searchSender.trim() ); - if (searchTime) - params.append("time", searchTime.trim()); + if (searchTimeValue) + params.append("time", searchTimeValue); if (searchLimit) params.append("limit", searchLimit); if (searchOffset) params.append("offset", searchOffset); + if (searchFormatValue) { + params.append("format", searchFormatValue); + responseFormat = searchFormatValue; + } break; + } } - // 添加参数到URL const apiUrl = params.toString() ? `${url}?${params.toString()}` : url; - - // 获取完整URL（包含域名部分） const fullUrl = window.location.origin + apiUrl; - - // 显示完整请求URL requestUrlContainer.textContent = fullUrl; resultWrapper.style.display = "block"; - - // 显示加载中 resultContainer.innerHTML = '
加载中
'; - // 发送请求 const response = await fetch(apiUrl); - if (!response.ok) { throw new Error( `HTTP error! Status: ${response.status}` ); } - // 获取响应内容类型 const contentType = - response.headers.get("content-type"); - let result; - - if ( - contentType && - contentType.includes("application/json") - ) { - // 如果是JSON，格式化显示 - result = await response.json(); - resultContainer.innerHTML = JSON.stringify( - result, - null, - 2 + response.headers.get("content-type") || ""; + if (contentType.includes("application/json")) { + const data = await response.json(); + const text = JSON.stringify(data, null, 2); + const highlighted = highlightPlainText( + text, + highlightTerms ); + resultContainer.innerHTML = `
${highlighted}
`; } else { - // 其他格式直接显示文本 - result = await response.text(); - resultContainer.innerHTML = result; + const textResult = await response.text(); + const expectsHtml = + (responseFormat || "").toLowerCase() === + "html" || contentType.includes("text/html"); + if (expectsHtml) { + resultContainer.innerHTML = textResult; + highlightHtmlContent( + resultContainer, + highlightTerms + ); + } else { + const highlighted = highlightPlainText( + textResult, + highlightTerms + ); + resultContainer.innerHTML = `
${highlighted}
`; + } } } catch (error) { resultContainer.innerHTML = ""; @@ -1093,6 +1185,112 @@ } }); + function escapeHtml(str) { + if (str == null) return ""; + return str + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + } + + function extractSearchTerms(raw) { + if (!raw) return []; + return Array.from( + new Set( + raw + .split(/[\s,，；;、]+/) + .map((term) => term.trim()) + .filter(Boolean) + ) + ); + } + + function buildHighlightPattern(terms) { + if (!terms || !terms.length) return ""; + const escaped = Array.from( + new Set( + terms + .map((term) => + term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + ) + .filter(Boolean) + ) + ).sort((a, b) => b.length - a.length); + if (!escaped.length) return ""; + return `(${escaped.join("|")})`; + } + + function highlightPlainText(text, terms) { + const pattern = buildHighlightPattern(terms); + if (!pattern) { + return escapeHtml(text); + } + const regex = new RegExp(pattern, "gi"); + let lastIndex = 0; + let highlighted = ""; + text.replace(regex, (match, _group, offset) => { + highlighted += escapeHtml(text.slice(lastIndex, offset)); + highlighted += `${escapeHtml( + match + )}`; + lastIndex = offset + match.length; + return match; + }); + highlighted += escapeHtml(text.slice(lastIndex)); + return highlighted; + } + + function highlightHtmlContent(container, terms) { + const pattern = buildHighlightPattern(terms); + if (!pattern) return; + const walker = document.createTreeWalker( + container, + NodeFilter.SHOW_TEXT, + null, + false + ); + const nodes = []; + while (walker.nextNode()) { + const node = walker.currentNode; + if (!node || !node.nodeValue || !node.nodeValue.trim()) + continue; + nodes.push(node); + } + nodes.forEach((textNode) => { + const text = textNode.nodeValue; + const regex = new RegExp(pattern, "gi"); + if (!regex.test(text)) { + return; + } + regex.lastIndex = 0; + const fragment = document.createDocumentFragment(); + let lastIndex = 0; + text.replace(regex, (match, _group, offset) => { + if (offset > lastIndex) { + fragment.appendChild( + document.createTextNode( + text.slice(lastIndex, offset) + ) + ); + } + const mark = document.createElement("mark"); + mark.className = "search-highlight"; + mark.textContent = match; + fragment.appendChild(mark); + lastIndex = offset + match.length; + return match; + }); + if (lastIndex < text.length) { + fragment.appendChild( + document.createTextNode(text.slice(lastIndex)) + ); + } + textNode.parentNode.replaceChild(fragment, textNode); + }); + } + const diaryDateInput = document.getElementById("diary-date"); if (diaryDateInput) { const today = new Date(); diff --git a/internal/chatlog/webhook/webhook.go b/internal/chatlog/webhook/webhook.go index 5e0d725..5042057 100644 --- a/internal/chatlog/webhook/webhook.go +++ b/internal/chatlog/webhook/webhook.go @@ -98,8 +98,7 @@ func NewGroup(ctx context.Context, group string, hooks []Webhook, delayMs int64) } func (g *Group) Callback(event fsnotify.Event) error { - // skip remove event - if !event.Op.Has(fsnotify.Create) { + if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename)) { return nil } diff --git a/internal/model/search.go b/internal/model/search.go index c48964b..f4a145e 100644 --- a/internal/model/search.go +++ b/internal/model/search.go @@ -6,7 +6,7 @@ import "time" // Start 和 End 为闭区间；如未提供则由调用方决定默认范围 // Limit/Offset 由调用链路在进入数据源前进行裁剪 // Sender 使用英文逗号分隔多个筛选条件 -// Talker 目前要求调用方传入具体会话（可多选，英文逗号分隔） +// Talker 可选：留空时后端会遍历所有会话；如需限定多个会话，使用英文逗号分隔 type SearchRequest struct { Query string `json:"query"` Talker string `json:"talker"` @@ -39,14 +39,25 @@ type SearchHit struct { // Limit / Offset 为实际生效的分页参数 // Hits 序列按相关度排序，命中数可能小于 limit（例如过滤后不足） type SearchResponse struct { - Total int `json:"total"` - Hits []*SearchHit `json:"hits"` - DurationMs int64 `json:"duration_ms"` - Limit int `json:"limit"` - Offset int `json:"offset"` - Query string `json:"query"` - Talker string `json:"talker"` - Sender string `json:"sender"` - Start time.Time `json:"start"` - End time.Time `json:"end"` + Total int `json:"total"` + Hits []*SearchHit `json:"hits"` + DurationMs int64 `json:"duration_ms"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Query string `json:"query"` + Talker string `json:"talker"` + Sender string `json:"sender"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Index *SearchIndexStatus `json:"index_status,omitempty"` +} + +// SearchIndexStatus 表示 Bleve 索引的构建状态 +type SearchIndexStatus struct { + Ready bool `json:"ready"` + InProgress bool `json:"in_progress"` + Progress float64 `json:"progress"` + LastStartedAt time.Time `json:"last_started_at"` + LastCompletedAt time.Time `json:"last_completed_at"` + LastError string `json:"last_error,omitempty"` } diff --git a/internal/wechatdb/datasource/darwinv3/datasource.go b/internal/wechatdb/datasource/darwinv3/datasource.go index 67f211b..9470c11 100644 --- a/internal/wechatdb/datasource/darwinv3/datasource.go +++ b/internal/wechatdb/datasource/darwinv3/datasource.go @@ -819,6 +819,10 @@ func (ds *DataSource) Close() error { return ds.dbm.Close() } +func (ds *DataSource) GetDatasetFingerprint(context.Context) (string, error) { + return ds.dbm.FingerprintForGroups(Message) +} + // GetAvatar returns not found for darwin v3 (no head image source known here) func (ds *DataSource) GetAvatar(ctx context.Context, username string, size string) (*model.Avatar, error) { return nil, errors.ErrAvatarNotFound diff --git a/internal/wechatdb/datasource/datasource.go b/internal/wechatdb/datasource/datasource.go index 0cac860..6211d9b 100644 --- a/internal/wechatdb/datasource/datasource.go +++ b/internal/wechatdb/datasource/datasource.go @@ -18,6 +18,7 @@ type DataSource interface { // 消息 GetMessages(ctx context.Context, startTime, endTime time.Time, talker string, sender string, keyword string, limit, offset int) ([]*model.Message, error) SearchMessages(ctx context.Context, req *model.SearchRequest) (*model.SearchResponse, error) + GetDatasetFingerprint(ctx context.Context) (string, error) // 联系人 GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error) diff --git a/internal/wechatdb/datasource/dbm/fingerprint.go b/internal/wechatdb/datasource/dbm/fingerprint.go new file mode 100644 index 0000000..619ea48 --- /dev/null +++ b/internal/wechatdb/datasource/dbm/fingerprint.go @@ -0,0 +1,83 @@ +package dbm + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// FingerprintForGroups calculates a stable fingerprint for the provided file groups. +// It inspects the current file list managed by DBManager, collects their size and +// modification timestamp, and hashes the aggregated metadata. If none of the +// requested groups contain files, an empty string is returned without error. +func (d *DBManager) FingerprintForGroups(names ...string) (string, error) { + if len(names) == 0 { + return "", nil + } + + type fileFingerprint struct { + group string + rel string + size int64 + mod int64 + } + + entries := make([]fileFingerprint, 0) + + for _, name := range names { + paths, err := d.GetDBPath(name) + if err != nil { + // When the underlying error indicates the files are simply missing, + // treat it as an empty group rather than a hard failure. + if strings.Contains(err.Error(), "db file not found") { + continue + } + return "", err + } + + for _, p := range paths { + info, statErr := os.Stat(p) + if statErr != nil { + if os.IsNotExist(statErr) { + continue + } + return "", statErr + } + + rel := p + if relPath, relErr := filepath.Rel(d.path, p); relErr == nil { + rel = relPath + } + rel = filepath.ToSlash(rel) + + entries = append(entries, fileFingerprint{ + group: name, + rel: rel, + size: info.Size(), + mod: info.ModTime().UnixNano(), + }) + } + } + + if len(entries) == 0 { + return "", nil + } + + sort.Slice(entries, func(i, j int) bool { + if entries[i].group == entries[j].group { + return entries[i].rel < entries[j].rel + } + return entries[i].group < entries[j].group + }) + + hasher := sha256.New() + for _, e := range entries { + fmt.Fprintf(hasher, "%s|%s|%d|%d;", e.group, e.rel, e.size, e.mod) + } + + return hex.EncodeToString(hasher.Sum(nil)), nil +} diff --git a/internal/wechatdb/datasource/v4/datasource.go b/internal/wechatdb/datasource/v4/datasource.go index a9ff3dc..355c985 100644 --- a/internal/wechatdb/datasource/v4/datasource.go +++ b/internal/wechatdb/datasource/v4/datasource.go @@ -644,6 +644,10 @@ func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchReque }, nil } +func (ds *DataSource) GetDatasetFingerprint(context.Context) (string, error) { + return ds.dbm.FingerprintForGroups(Message) +} + // 联系人 func (ds *DataSource) GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error) { var query string diff --git a/internal/wechatdb/datasource/windowsv3/datasource.go b/internal/wechatdb/datasource/windowsv3/datasource.go index 85422a6..77e2b89 100644 --- a/internal/wechatdb/datasource/windowsv3/datasource.go +++ b/internal/wechatdb/datasource/windowsv3/datasource.go @@ -9,6 +9,7 @@ import ( "regexp" "sort" "strings" + "sync" "time" "github.com/fsnotify/fsnotify" @@ -28,6 +29,8 @@ const ( Video = "video" File = "file" Voice = "voice" + + talkerCacheTTL = 30 * time.Second ) var Groups = []*dbm.Group{ @@ -78,6 +81,10 @@ type DataSource struct { // 消息数据库信息 messageInfos []MessageDBInfo + + talkerCacheMu sync.RWMutex + talkerCache []string + talkerCacheExpiry time.Time } // New 创建一个新的 WindowsV3DataSource @@ -101,12 +108,21 @@ func New(path string) (*DataSource, error) { } ds.dbm.AddCallback(Message, func(event fsnotify.Event) error { - if !event.Op.Has(fsnotify.Create) { + if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename)) { return nil } if err := ds.initMessageDbs(); err != nil { log.Err(err).Msgf("Failed to reinitialize message DBs: %s", event.Name) } + ds.invalidateTalkerCache() + return nil + }) + + ds.dbm.AddCallback(Contact, func(event fsnotify.Event) error { + if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename) || event.Op.Has(fsnotify.Remove)) { + return nil + } + ds.invalidateTalkerCache() return nil }) @@ -214,6 +230,7 @@ func (ds *DataSource) initMessageDbs() error { return nil } ds.messageInfos = infos + ds.invalidateTalkerCache() return nil } @@ -429,7 +446,25 @@ func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchReque talkers := util.Str2List(req.Talker, ",") if len(talkers) == 0 { - return nil, errors.ErrTalkerEmpty + var err error + talkers, err = ds.collectAllTalkers(ctx) + if err != nil { + return nil, err + } + } + if len(talkers) == 0 { + return &model.SearchResponse{ + Total: 0, + Hits: []*model.SearchHit{}, + DurationMs: 0, + Limit: req.Limit, + Offset: req.Offset, + Query: req.Query, + Talker: req.Talker, + Sender: req.Sender, + Start: req.Start, + End: req.End, + }, nil } limit := req.Limit @@ -676,6 +711,117 @@ func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchReque }, nil } +func (ds *DataSource) GetDatasetFingerprint(context.Context) (string, error) { + return ds.dbm.FingerprintForGroups(Message) +} + +func (ds *DataSource) getCachedTalkers() []string { + ds.talkerCacheMu.RLock() + if ds.talkerCacheExpiry.IsZero() || time.Now().After(ds.talkerCacheExpiry) { + ds.talkerCacheMu.RUnlock() + return nil + } + cached := append([]string(nil), ds.talkerCache...) + ds.talkerCacheMu.RUnlock() + return cached +} + +func (ds *DataSource) cacheTalkers(talkers []string) { + copySlice := append([]string(nil), talkers...) + ds.talkerCacheMu.Lock() + ds.talkerCache = copySlice + ds.talkerCacheExpiry = time.Now().Add(talkerCacheTTL) + ds.talkerCacheMu.Unlock() +} + +func (ds *DataSource) invalidateTalkerCache() { + ds.talkerCacheMu.Lock() + ds.talkerCache = nil + ds.talkerCacheExpiry = time.Time{} + ds.talkerCacheMu.Unlock() +} + +func (ds *DataSource) collectAllTalkers(ctx context.Context) ([]string, error) { + if cached := ds.getCachedTalkers(); cached != nil { + return cached, nil + } + + talkerSet := make(map[string]struct{}) + talkers := make([]string, 0) + + add := func(name string) { + name = strings.TrimSpace(name) + if name == "" { + return + } + if _, exists := talkerSet[name]; exists { + return + } + talkerSet[name] = struct{}{} + talkers = append(talkers, name) + } + + if db, err := ds.dbm.GetDB(Contact); err == nil && db != nil { + rows, err := db.QueryContext(ctx, `SELECT strUsrName FROM Session WHERE IFNULL(strUsrName,'') <> ''`) + if err == nil { + for rows.Next() { + if err := ctx.Err(); err != nil { + rows.Close() + return nil, err + } + var username string + if err := rows.Scan(&username); err != nil { + log.Debug().Err(err).Msg("scan session talker failed") + continue + } + add(username) + } + if err := rows.Err(); err != nil { + log.Debug().Err(err).Msg("iterate session talkers failed") + } + rows.Close() + } else { + log.Debug().Err(err).Msg("query session talkers failed") + } + + if len(talkers) == 0 { + rows, err = db.QueryContext(ctx, `SELECT UserName FROM Contact WHERE IFNULL(UserName,'') <> ''`) + if err == nil { + for rows.Next() { + if err := ctx.Err(); err != nil { + rows.Close() + return nil, err + } + var username string + if err := rows.Scan(&username); err != nil { + log.Debug().Err(err).Msg("scan contact talker failed") + continue + } + add(username) + } + if err := rows.Err(); err != nil { + log.Debug().Err(err).Msg("iterate contact talkers failed") + } + rows.Close() + } else { + log.Debug().Err(err).Msg("query contact talkers failed") + } + } + } + + if len(talkers) == 0 { + for _, info := range ds.messageInfos { + for username := range info.TalkerMap { + add(username) + } + } + } + + sort.Strings(talkers) + ds.cacheTalkers(talkers) + return talkers, nil +} + // GetContacts 实现获取联系人信息的方法 func (ds *DataSource) GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error) { var query string diff --git a/internal/wechatdb/indexer/indexer.go b/internal/wechatdb/indexer/indexer.go new file mode 100644 index 0000000..6e980c8 --- /dev/null +++ b/internal/wechatdb/indexer/indexer.go @@ -0,0 +1,482 @@ +package indexer + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search/query" + + "github.com/sjzar/chatlog/internal/model" +) + +const ( + runtimeIndexVersionKey = "chatlog_index_version" + runtimeIndexVersion = "1" + fingerprintKey = "chatlog_index_fingerprint" + lastBuiltKey = "chatlog_index_last_built" +) + +// SearchHit represents a single Bleve search hit mapped back to chatlog's domain model. +type SearchHit struct { + Message *model.Message + Snippet string + Score float64 +} + +// Index wraps a Bleve index with concurrency control and helper utilities. +type Index struct { + mu sync.RWMutex + idx bleve.Index + path string +} + +// Open opens an existing index at the given path or creates a new one with the default mapping. +func Open(path string) (*Index, error) { + parent := filepath.Dir(path) + if err := os.MkdirAll(parent, 0o755); err != nil { + return nil, fmt.Errorf("create index parent dir: %w", err) + } + + var ( + idx bleve.Index + err error + ) + if _, statErr := os.Stat(path); statErr == nil { + idx, err = bleve.Open(path) + if err != nil { + return nil, fmt.Errorf("open bleve index: %w", err) + } + } else if errors.Is(statErr, os.ErrNotExist) { + mapping, mapErr := buildMapping() + if mapErr != nil { + return nil, mapErr + } + idx, err = bleve.New(path, mapping) + if err != nil { + return nil, fmt.Errorf("create bleve index: %w", err) + } + } else { + return nil, fmt.Errorf("stat index: %w", statErr) + } + + return &Index{idx: idx, path: path}, nil +} + +// Close closes the underlying Bleve index. +func (i *Index) Close() error { + if i == nil { + return nil + } + i.mu.Lock() + defer i.mu.Unlock() + if i.idx == nil { + return nil + } + return i.idx.Close() +} + +// Reset drops all existing data and recreates the index with the default mapping. +func (i *Index) Reset() error { + i.mu.Lock() + defer i.mu.Unlock() + + if i.idx != nil { + _ = i.idx.Close() + } + if err := os.RemoveAll(i.path); err != nil { + return fmt.Errorf("remove index dir: %w", err) + } + mapping, err := buildMapping() + if err != nil { + return err + } + idx, err := bleve.New(i.path, mapping) + if err != nil { + return fmt.Errorf("recreate bleve index: %w", err) + } + i.idx = idx + return nil +} + +// SetMetadata stores an internal key/value pair within the index (not searchable). +func (i *Index) SetMetadata(key string, value []byte) error { + i.mu.Lock() + defer i.mu.Unlock() + if i.idx == nil { + return errors.New("index not initialized") + } + return i.idx.SetInternal([]byte(key), value) +} + +// GetMetadata retrieves an internal key/value pair. +func (i *Index) GetMetadata(key string) ([]byte, error) { + i.mu.RLock() + defer i.mu.RUnlock() + if i.idx == nil { + return nil, errors.New("index not initialized") + } + return i.idx.GetInternal([]byte(key)) +} + +// IndexMessages indexes a batch of messages using Bleve batches for efficiency. +func (i *Index) IndexMessages(messages []*model.Message) error { + if len(messages) == 0 { + return nil + } + i.mu.Lock() + defer i.mu.Unlock() + if i.idx == nil { + return errors.New("index not initialized") + } + + batch := i.idx.NewBatch() + const batchSize = 250 + for idx, msg := range messages { + doc, err := newDocument(msg) + if err != nil { + return err + } + if err := batch.Index(doc.ID, doc); err != nil { + return fmt.Errorf("batch index: %w", err) + } + if (idx+1)%batchSize == 0 { + if err := i.idx.Batch(batch); err != nil { + return fmt.Errorf("flush batch: %w", err) + } + batch = i.idx.NewBatch() + } + } + + if batch.Size() > 0 { + if err := i.idx.Batch(batch); err != nil { + return fmt.Errorf("flush final batch: %w", err) + } + } + + return nil +} + +// Search executes a Bleve search over the indexed content applying optional filters. +func (i *Index) Search(req *model.SearchRequest, talkers []string, senders []string, startUnix, endUnix int64, offset, limit int) ([]*SearchHit, int, error) { + if req == nil { + return nil, 0, errors.New("search request is nil") + } + + queryObj := buildQuery(req.Query, talkers, senders, startUnix, endUnix) + if queryObj == nil { + return []*SearchHit{}, 0, nil + } + + i.mu.RLock() + idx := i.idx + i.mu.RUnlock() + if idx == nil { + return nil, 0, errors.New("index not initialized") + } + + if limit <= 0 { + limit = 20 + } + if limit > 200 { + limit = 200 + } + if offset < 0 { + offset = 0 + } + + searchRequest := bleve.NewSearchRequestOptions(queryObj, limit, offset, false) + searchRequest.Highlight = bleve.NewHighlightWithStyle("html") + searchRequest.Fields = []string{"message_json"} + searchRequest.IncludeLocations = false + + result, err := idx.Search(searchRequest) + if err != nil { + return nil, 0, fmt.Errorf("bleve search: %w", err) + } + + hits := make([]*SearchHit, 0, len(result.Hits)) + for _, hit := range result.Hits { + messageJSON, ok := hit.Fields["message_json"].(string) + if !ok || messageJSON == "" { + continue + } + var msg model.Message + if err := json.Unmarshal([]byte(messageJSON), &msg); err != nil { + return nil, 0, fmt.Errorf("decode message: %w", err) + } + + snippet := "" + if frags, ok := hit.Fragments["content"]; ok && len(frags) > 0 { + snippet = strings.Join(frags, " … ") + } + + hits = append(hits, &SearchHit{ + Message: &msg, + Snippet: snippet, + Score: hit.Score, + }) + } + + return hits, int(result.Total), nil +} + +// Document representation stored inside Bleve. +type document struct { + ID string `json:"id"` + Talker string `json:"talker"` + Sender string `json:"sender"` + Unix int64 `json:"unix"` + Seq int64 `json:"seq"` + Content string `json:"content"` + MessageJSON string `json:"message_json"` +} + +func newDocument(msg *model.Message) (*document, error) { + if msg == nil { + return nil, errors.New("nil message") + } + content := msg.PlainTextContent() + messageJSON, err := json.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("marshal message: %w", err) + } + return &document{ + ID: fmt.Sprintf("%s:%d", msg.Talker, msg.Seq), + Talker: msg.Talker, + Sender: msg.Sender, + Unix: msg.Time.Unix(), + Seq: msg.Seq, + Content: content, + MessageJSON: string(messageJSON), + }, nil +} + +func buildMapping() (*mapping.IndexMappingImpl, error) { + indexMapping := bleve.NewIndexMapping() + indexMapping.TypeField = "type" + indexMapping.DefaultAnalyzer = "standard" + + docMapping := mapping.NewDocumentMapping() + + contentField := mapping.NewTextFieldMapping() + contentField.Analyzer = "standard" + contentField.Store = false + docMapping.AddFieldMappingsAt("content", contentField) + + talkerField := mapping.NewTextFieldMapping() + talkerField.Analyzer = "keyword" + talkerField.Store = true + talkerField.IncludeInAll = false + docMapping.AddFieldMappingsAt("talker", talkerField) + + senderField := mapping.NewTextFieldMapping() + senderField.Analyzer = "keyword" + senderField.Store = true + senderField.IncludeInAll = false + docMapping.AddFieldMappingsAt("sender", senderField) + + unixField := mapping.NewNumericFieldMapping() + unixField.Store = true + docMapping.AddFieldMappingsAt("unix", unixField) + + seqField := mapping.NewNumericFieldMapping() + seqField.Store = true + docMapping.AddFieldMappingsAt("seq", seqField) + + messageField := mapping.NewTextFieldMapping() + messageField.Analyzer = "keyword" + messageField.Store = true + messageField.Index = false + docMapping.AddFieldMappingsAt("message_json", messageField) + + indexMapping.DefaultMapping = docMapping + + return indexMapping, nil +} + +func buildQuery(input string, talkers []string, senders []string, startUnix, endUnix int64) query.Query { + contentQuery := buildContentQuery(input) + + var must []query.Query + if contentQuery != nil { + must = append(must, contentQuery) + } + + if len(talkers) > 0 { + must = append(must, buildTermsFilter("talker", talkers)) + } + if len(senders) > 0 { + must = append(must, buildTermsFilter("sender", senders)) + } + if startUnix > 0 || endUnix > 0 { + var minPtr, maxPtr *float64 + if startUnix > 0 { + min := float64(startUnix) + minPtr = &min + } + if endUnix > 0 { + max := float64(endUnix) + maxPtr = &max + } + rangeQuery := query.NewNumericRangeQuery(minPtr, maxPtr) + rangeQuery.SetField("unix") + must = append(must, rangeQuery) + } + + if len(must) == 0 { + return nil + } + + if len(must) == 1 { + return must[0] + } + + return query.NewConjunctionQuery(must) +} + +func buildContentQuery(input string) query.Query { + s := strings.TrimSpace(input) + if s == "" { + return nil + } + + upper := strings.ToUpper(s) + advanced := strings.ContainsAny(s, "\"'*()") || + strings.Contains(upper, " AND ") || + strings.Contains(upper, " OR ") || + strings.Contains(upper, " NEAR ") || + strings.HasPrefix(upper, "NOT ") + + if advanced { + return query.NewQueryStringQuery(s) + } + + tokens := strings.Fields(s) + if len(tokens) == 0 { + return nil + } + + if len(tokens) == 1 { + mq := query.NewMatchQuery(tokens[0]) + mq.SetField("content") + return mq + } + + conj := make([]query.Query, 0, len(tokens)) + for _, token := range tokens { + t := strings.TrimSpace(token) + if t == "" { + continue + } + mq := query.NewMatchQuery(t) + mq.SetField("content") + conj = append(conj, mq) + } + + if len(conj) == 0 { + return nil + } + if len(conj) == 1 { + return conj[0] + } + return query.NewConjunctionQuery(conj) +} + +func buildTermsFilter(field string, values []string) query.Query { + sanitized := make([]query.Query, 0, len(values)) + for _, val := range values { + trimmed := strings.TrimSpace(val) + if trimmed == "" { + continue + } + tq := query.NewTermQuery(trimmed) + tq.SetField(field) + sanitized = append(sanitized, tq) + } + if len(sanitized) == 0 { + return nil + } + if len(sanitized) == 1 { + return sanitized[0] + } + return query.NewDisjunctionQuery(sanitized) +} + +// EnsureVersion ensures the stored index version matches current runtime version. +func (i *Index) EnsureVersion() (bool, error) { + current, _ := i.GetMetadata(runtimeIndexVersionKey) + if string(current) == runtimeIndexVersion { + return true, nil + } + if err := i.SetMetadata(runtimeIndexVersionKey, []byte(runtimeIndexVersion)); err != nil { + return false, err + } + return false, nil +} + +// UpdateFingerprint persists the dataset fingerprint into index metadata. +func (i *Index) UpdateFingerprint(fp string) error { + return i.SetMetadata(fingerprintKey, []byte(fp)) +} + +// Fingerprint reads the stored dataset fingerprint from index metadata. +func (i *Index) Fingerprint() string { + b, err := i.GetMetadata(fingerprintKey) + if err != nil { + return "" + } + return string(b) +} + +// FingerprintMatches reports whether the stored dataset fingerprint equals the provided value. +func (i *Index) FingerprintMatches(fp string) bool { + if fp == "" { + return false + } + return i.Fingerprint() == fp +} + +// EnsureFingerprint compares the stored fingerprint with the provided value. +// It returns true when the fingerprint already matches. When it differs, the +// metadata is updated to the new fingerprint and false is returned to signal +// callers that a rebuild just occurred. +func (i *Index) EnsureFingerprint(fp string) (bool, error) { + if fp == "" { + return false, nil + } + current := i.Fingerprint() + if current == fp { + return true, nil + } + if err := i.UpdateFingerprint(fp); err != nil { + return false, err + } + return false, nil +} + +// UpdateLastBuilt 记录最近一次索引构建完成时间（Unix 秒） +func (i *Index) UpdateLastBuilt(t time.Time) error { + return i.SetMetadata(lastBuiltKey, []byte(strconv.FormatInt(t.Unix(), 10))) +} + +// LastBuilt 返回最近一次索引构建完成时间 +func (i *Index) LastBuilt() time.Time { + b, err := i.GetMetadata(lastBuiltKey) + if err != nil || len(b) == 0 { + return time.Time{} + } + sec, parseErr := strconv.ParseInt(string(b), 10, 64) + if parseErr != nil { + return time.Time{} + } + return time.Unix(sec, 0) +} diff --git a/internal/wechatdb/repository/repository.go b/internal/wechatdb/repository/repository.go index 83afaa4..f1163f6 100644 --- a/internal/wechatdb/repository/repository.go +++ b/internal/wechatdb/repository/repository.go @@ -86,7 +86,7 @@ func (r *Repository) initCache(ctx context.Context) error { } func (r *Repository) contactCallback(event fsnotify.Event) error { - if !event.Op.Has(fsnotify.Create) { + if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename) || event.Op.Has(fsnotify.Remove)) { return nil } if err := r.initContactCache(context.Background()); err != nil { @@ -96,7 +96,7 @@ func (r *Repository) contactCallback(event fsnotify.Event) error { } func (r *Repository) chatroomCallback(event fsnotify.Event) error { - if !event.Op.Has(fsnotify.Create) { + if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename) || event.Op.Has(fsnotify.Remove)) { return nil } if err := r.initChatRoomCache(context.Background()); err != nil {