!refactor(search): Use Bleve in replace of SQLite FTS5
This commit is contained in:
Symb0x76
@@ -4,6 +4,7 @@ go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/Eyevinn/mp4ff v0.49.0
|
||||
github.com/blevesearch/bleve/v2 v2.4.2
|
||||
github.com/cespare/xxhash v1.1.0
|
||||
github.com/fsnotify/fsnotify v1.9.0
|
||||
github.com/gdamore/tcell/v2 v2.8.1
|
||||
@@ -29,7 +30,26 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/RoaringBitmap/roaring v1.9.3 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bits-and-blooms/bitset v1.12.0 // indirect
|
||||
github.com/blevesearch/bleve_index_api v1.1.10 // indirect
|
||||
github.com/blevesearch/geo v0.1.20 // indirect
|
||||
github.com/blevesearch/go-faiss v1.0.20 // indirect
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
|
||||
github.com/blevesearch/gtreap v0.1.1 // indirect
|
||||
github.com/blevesearch/mmap-go v1.0.4 // indirect
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.2.15 // indirect
|
||||
github.com/blevesearch/segment v0.9.1 // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
|
||||
github.com/blevesearch/vellum v1.0.10 // indirect
|
||||
github.com/blevesearch/zapx/v11 v11.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v12 v12.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v13 v13.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v14 v14.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v15 v15.3.13 // indirect
|
||||
github.com/blevesearch/zapx/v16 v16.1.5 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/bytedance/sonic v1.14.0 // indirect
|
||||
github.com/bytedance/sonic/loader v0.3.0 // indirect
|
||||
@@ -44,6 +64,9 @@ require (
|
||||
github.com/go-playground/validator/v10 v10.27.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/golang/protobuf v1.5.0 // indirect
|
||||
github.com/golang/snappy v0.0.1 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/invopop/jsonschema v0.13.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
@@ -57,6 +80,7 @@ require (
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mschoch/smat v0.2.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
@@ -73,6 +97,7 @@ require (
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
go.etcd.io/bbolt v1.3.7 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/net v0.43.0 // indirect
|
||||
golang.org/x/term v0.34.0 // indirect
|
||||
|
||||
@@ -2,8 +2,48 @@ github.com/Eyevinn/mp4ff v0.49.0 h1:00eRg5/KwcLGWUbv+hlifldf74qg46G1IxoWXHrgDvw=
|
||||
github.com/Eyevinn/mp4ff v0.49.0/go.mod h1:hJNUUqOBryLAzUW9wpCJyw2HaI+TCd2rUPhafoS5lgg=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM=
|
||||
github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
|
||||
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/blevesearch/bleve/v2 v2.4.2 h1:NooYP1mb3c0StkiY9/xviiq2LGSaE8BQBCc/pirMx0U=
|
||||
github.com/blevesearch/bleve/v2 v2.4.2/go.mod h1:ATNKj7Yl2oJv/lGuF4kx39bST2dveX6w0th2FFYLkc8=
|
||||
github.com/blevesearch/bleve_index_api v1.1.10 h1:PDLFhVjrjQWr6jCuU7TwlmByQVCSEURADHdCqVS9+g0=
|
||||
github.com/blevesearch/bleve_index_api v1.1.10/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8=
|
||||
github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
|
||||
github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
|
||||
github.com/blevesearch/go-faiss v1.0.20 h1:AIkdTQFWuZ5LQmKQSebgMR4RynGNw8ZseJXaan5kvtI=
|
||||
github.com/blevesearch/go-faiss v1.0.20/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
|
||||
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
|
||||
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
|
||||
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
|
||||
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.2.15 h1:prV17iU/o+A8FiZi9MXmqbagd8I0bCqM7OKUYPbnb5Y=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.2.15/go.mod h1:db0cmP03bPNadXrCDuVkKLV6ywFSiRgPFT1YVrestBc=
|
||||
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
|
||||
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
|
||||
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
|
||||
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
|
||||
github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
|
||||
github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
|
||||
github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
|
||||
github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
|
||||
github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
|
||||
github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
|
||||
github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
|
||||
github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
|
||||
github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
|
||||
github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
|
||||
github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
|
||||
github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
|
||||
github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi/AUHjU=
|
||||
github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8=
|
||||
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
|
||||
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
|
||||
github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ=
|
||||
@@ -53,6 +93,13 @@ github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
@@ -103,6 +150,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
|
||||
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
@@ -176,6 +225,8 @@ github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT0
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
|
||||
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
|
||||
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
|
||||
golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
@@ -255,6 +306,8 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
|
||||
google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
+236
-10
@@ -722,6 +722,7 @@ func (s *Service) handleSearch(c *gin.Context) {
|
||||
End string `form:"end"`
|
||||
Limit int `form:"limit"`
|
||||
Offset int `form:"offset"`
|
||||
Format string `form:"format"`
|
||||
}{}
|
||||
|
||||
if err := c.BindQuery(¶ms); err != nil {
|
||||
@@ -730,16 +731,8 @@ func (s *Service) handleSearch(c *gin.Context) {
|
||||
}
|
||||
|
||||
query := strings.TrimSpace(params.Query)
|
||||
if query == "" {
|
||||
errors.Err(c, errors.InvalidArg("q"))
|
||||
return
|
||||
}
|
||||
|
||||
talker := strings.TrimSpace(params.Talker)
|
||||
if talker == "" {
|
||||
errors.Err(c, errors.ErrTalkerEmpty)
|
||||
return
|
||||
}
|
||||
|
||||
limit := params.Limit
|
||||
if limit <= 0 {
|
||||
@@ -818,7 +811,163 @@ func (s *Service) handleSearch(c *gin.Context) {
|
||||
resp.Limit = limit
|
||||
resp.Offset = offset
|
||||
|
||||
c.JSON(http.StatusOK, resp)
|
||||
format := strings.ToLower(strings.TrimSpace(params.Format))
|
||||
terms := extractQueryTerms(resp.Query)
|
||||
|
||||
switch format {
|
||||
case "html":
|
||||
c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
c.Writer.WriteString(`<html><head><meta charset="utf-8"><title>Search Result</title><style>body{font-family:Arial,Helvetica,sans-serif;font-size:14px;line-height:1.5;background:#f8f9fb;padding:24px;color:#2c3e50;}h1{margin:0 0 16px;font-size:22px;}h2{margin:24px 0 12px;font-size:18px;}p.meta{margin:4px 0;color:#5f6c7b;} .summary{background:#fff;padding:18px;border-radius:10px;box-shadow:0 1px 4px rgba(18,38,63,0.08);margin-bottom:18px;} .result-list{display:flex;flex-direction:column;gap:16px;} .hit{background:#fff;border-radius:10px;padding:16px 18px;box-shadow:0 1px 4px rgba(18,38,63,0.08);} .hit-header{display:flex;justify-content:space-between;align-items:center;margin-bottom:8px;font-size:13px;color:#5f6c7b;} .hit-header .talker{font-weight:600;color:#2c3e50;} .hit-header .score{font-family:monospace;color:#a0aec0;} .snippet{margin-bottom:10px;font-size:13px;color:#4a5568;padding:10px;border-left:3px solid #3498db;background:rgba(52,152,219,0.08);border-radius:6px;} .snippet code{background:rgba(27,31,35,0.05);padding:2px 4px;border-radius:4px;} .msg-content pre{white-space:pre-wrap;word-break:break-word;margin:0;font-family:"SFMono-Regular",Consolas,"Liberation Mono",Menlo,monospace;font-size:13px;color:#1f2933;} .meta-row{display:flex;flex-wrap:wrap;gap:12px;margin-bottom:8px;font-size:13px;color:#5f6c7b;} .meta-row span{display:inline-flex;align-items:center;gap:4px;} .search-highlight{background:#fff3b0;color:inherit;padding:0 2px;border-radius:3px;} .empty{padding:28px;text-align:center;color:#768390;background:#fff;border-radius:10px;box-shadow:0 1px 4px rgba(18,38,63,0.08);} a.media{color:#2c3e50;text-decoration:none;border-bottom:1px dashed #2c3e50;} a.media:hover{color:#0f4c81;} </style></head><body>`)
|
||||
c.Writer.WriteString("<div class=\"summary\">")
|
||||
c.Writer.WriteString("<h1>搜索结果</h1>")
|
||||
if resp.Query != "" {
|
||||
c.Writer.WriteString("<p class=\"meta\"><strong>关键词：</strong>" + template.HTMLEscapeString(resp.Query) + "</p>")
|
||||
}
|
||||
talkerLabel := "全部会话"
|
||||
if resp.Talker != "" {
|
||||
talkerLabel = template.HTMLEscapeString(resp.Talker)
|
||||
}
|
||||
c.Writer.WriteString("<p class=\"meta\"><strong>会话：</strong>" + talkerLabel + "</p>")
|
||||
if resp.Sender != "" {
|
||||
c.Writer.WriteString("<p class=\"meta\"><strong>发送者：</strong>" + template.HTMLEscapeString(resp.Sender) + "</p>")
|
||||
}
|
||||
timeLabel := "不限"
|
||||
if !resp.Start.IsZero() && !resp.End.IsZero() {
|
||||
timeLabel = resp.Start.Format("2006-01-02 15:04:05") + " ~ " + resp.End.Format("2006-01-02 15:04:05")
|
||||
} else if !resp.Start.IsZero() {
|
||||
timeLabel = ">= " + resp.Start.Format("2006-01-02 15:04:05")
|
||||
} else if !resp.End.IsZero() {
|
||||
timeLabel = "<= " + resp.End.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
c.Writer.WriteString("<p class=\"meta\"><strong>时间范围：</strong>" + template.HTMLEscapeString(timeLabel) + "</p>")
|
||||
c.Writer.WriteString(fmt.Sprintf("<p class=\"meta\"><strong>命中条数：</strong>%d（本页 %d 条）</p>", resp.Total, len(resp.Hits)))
|
||||
c.Writer.WriteString("</div>")
|
||||
|
||||
if len(resp.Hits) == 0 {
|
||||
c.Writer.WriteString("<div class=\"empty\">暂无搜索结果</div>")
|
||||
} else {
|
||||
c.Writer.WriteString("<div class=\"result-list\">")
|
||||
for idx, hit := range resp.Hits {
|
||||
if hit == nil || hit.Message == nil {
|
||||
continue
|
||||
}
|
||||
msg := hit.Message
|
||||
msg.SetContent("host", c.Request.Host)
|
||||
talkerDisplay := msg.Talker
|
||||
if msg.TalkerName != "" {
|
||||
talkerDisplay = fmt.Sprintf("%s (%s)", msg.TalkerName, msg.Talker)
|
||||
}
|
||||
senderDisplay := msg.Sender
|
||||
if msg.IsSelf {
|
||||
senderDisplay = "我"
|
||||
}
|
||||
if msg.SenderName != "" {
|
||||
senderDisplay = fmt.Sprintf("%s(%s)", msg.SenderName, msg.Sender)
|
||||
}
|
||||
c.Writer.WriteString("<div class=\"hit\">")
|
||||
c.Writer.WriteString("<div class=\"hit-header\"><span class=\"talker\">#" + fmt.Sprintf("%d", idx+1) + " · " + template.HTMLEscapeString(talkerDisplay) + "</span>")
|
||||
if hit.Score > 0 {
|
||||
c.Writer.WriteString("<span class=\"score\">score: " + fmt.Sprintf("%.4f", hit.Score) + "</span>")
|
||||
}
|
||||
c.Writer.WriteString("</div>")
|
||||
c.Writer.WriteString("<div class=\"meta-row\"><span>时间：" + template.HTMLEscapeString(msg.Time.Format("2006-01-02 15:04:05")) + "</span><span>发送者：" + template.HTMLEscapeString(senderDisplay) + "</span></div>")
|
||||
if snippet := strings.TrimSpace(hit.Snippet); snippet != "" {
|
||||
h := highlightPlainTextToHTML(snippet, terms)
|
||||
h = strings.ReplaceAll(h, "\n", "<br/>")
|
||||
c.Writer.WriteString("<div class=\"snippet\">" + h + "</div>")
|
||||
}
|
||||
c.Writer.WriteString("<div class=\"msg-content\"><pre>" + messageHTMLPlaceholder(msg) + "</pre></div>")
|
||||
c.Writer.WriteString("</div>")
|
||||
}
|
||||
c.Writer.WriteString("</div>")
|
||||
}
|
||||
c.Writer.WriteString(previewHTMLSnippet)
|
||||
c.Writer.WriteString("</body></html>")
|
||||
return
|
||||
case "text", "plain":
|
||||
c.Writer.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
c.Writer.Header().Set("Cache-Control", "no-cache")
|
||||
c.Writer.Header().Set("Connection", "keep-alive")
|
||||
fmt.Fprintf(c.Writer, "关键词: %s\n", resp.Query)
|
||||
talkerLabel := resp.Talker
|
||||
if talkerLabel == "" {
|
||||
talkerLabel = "全部会话"
|
||||
}
|
||||
fmt.Fprintf(c.Writer, "会话: %s\n", talkerLabel)
|
||||
if resp.Sender != "" {
|
||||
fmt.Fprintf(c.Writer, "发送者: %s\n", resp.Sender)
|
||||
}
|
||||
switch {
|
||||
case !resp.Start.IsZero() && !resp.End.IsZero():
|
||||
fmt.Fprintf(c.Writer, "时间: %s ~ %s\n", resp.Start.Format("2006-01-02 15:04:05"), resp.End.Format("2006-01-02 15:04:05"))
|
||||
case !resp.Start.IsZero():
|
||||
fmt.Fprintf(c.Writer, "时间: >= %s\n", resp.Start.Format("2006-01-02 15:04:05"))
|
||||
case !resp.End.IsZero():
|
||||
fmt.Fprintf(c.Writer, "时间: <= %s\n", resp.End.Format("2006-01-02 15:04:05"))
|
||||
default:
|
||||
fmt.Fprintln(c.Writer, "时间: 不限")
|
||||
}
|
||||
fmt.Fprintf(c.Writer, "总命中: %d, 本页: %d\n", resp.Total, len(resp.Hits))
|
||||
fmt.Fprintln(c.Writer, strings.Repeat("-", 60))
|
||||
for idx, hit := range resp.Hits {
|
||||
if hit == nil || hit.Message == nil {
|
||||
continue
|
||||
}
|
||||
msg := hit.Message
|
||||
msg.SetContent("host", c.Request.Host)
|
||||
title := msg.Talker
|
||||
if msg.TalkerName != "" {
|
||||
title = fmt.Sprintf("%s (%s)", msg.TalkerName, msg.Talker)
|
||||
}
|
||||
sender := msg.Sender
|
||||
if msg.IsSelf {
|
||||
sender = "我"
|
||||
}
|
||||
if msg.SenderName != "" {
|
||||
sender = fmt.Sprintf("%s(%s)", msg.SenderName, msg.Sender)
|
||||
}
|
||||
fmt.Fprintf(c.Writer, "[%d] %s @ %s\n", idx+1, msg.Time.Format("2006-01-02 15:04:05"), title)
|
||||
fmt.Fprintf(c.Writer, "发送者: %s\n", sender)
|
||||
fmt.Fprintf(c.Writer, "%s\n", msg.PlainTextContent())
|
||||
if snippet := strings.TrimSpace(hit.Snippet); snippet != "" {
|
||||
fmt.Fprintf(c.Writer, "Snippet: %s\n", snippet)
|
||||
}
|
||||
fmt.Fprintln(c.Writer, strings.Repeat("-", 60))
|
||||
}
|
||||
return
|
||||
case "csv":
|
||||
c.Writer.Header().Set("Content-Type", "text/csv; charset=utf-8")
|
||||
c.Writer.Header().Set("Cache-Control", "no-cache")
|
||||
c.Writer.Header().Set("Connection", "keep-alive")
|
||||
c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=search_%s.csv", time.Now().Format("20060102_150405")))
|
||||
csvWriter := csv.NewWriter(c.Writer)
|
||||
csvWriter.Write([]string{"Seq", "Time", "Talker", "TalkerName", "Sender", "SenderName", "Content", "Snippet"})
|
||||
for _, hit := range resp.Hits {
|
||||
if hit == nil || hit.Message == nil {
|
||||
continue
|
||||
}
|
||||
msg := hit.Message
|
||||
msg.SetContent("host", c.Request.Host)
|
||||
csvWriter.Write([]string{
|
||||
fmt.Sprintf("%d", msg.Seq),
|
||||
msg.Time.Format("2006-01-02 15:04:05"),
|
||||
msg.Talker,
|
||||
msg.TalkerName,
|
||||
msg.Sender,
|
||||
msg.SenderName,
|
||||
msg.PlainTextContent(),
|
||||
strings.ReplaceAll(hit.Snippet, "\n", " "),
|
||||
})
|
||||
}
|
||||
csvWriter.Flush()
|
||||
return
|
||||
case "json", "":
|
||||
c.JSON(http.StatusOK, resp)
|
||||
return
|
||||
default:
|
||||
c.JSON(http.StatusOK, resp)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) handleChatlog(c *gin.Context) {
|
||||
@@ -1501,7 +1650,10 @@ func (s *Service) HandleVoice(c *gin.Context, data []byte) {
|
||||
}
|
||||
|
||||
// 统一占位符：将 PlainTextContent 里形如  或 [标签](url) 的模式转成超链接形式，仅显示 [标签]。
|
||||
var placeholderPattern = regexp.MustCompile(`!?\[([^\]]+)\]\((https?://[^)]+)\)`)
|
||||
var (
|
||||
placeholderPattern = regexp.MustCompile(`!?\[([^\]]+)\]\((https?://[^)]+)\)`)
|
||||
searchTermSplitPattern = regexp.MustCompile(`[\s,，；;、]+`)
|
||||
)
|
||||
|
||||
func messageHTMLPlaceholder(m *model.Message) string {
|
||||
content := m.PlainTextContent()
|
||||
@@ -1536,3 +1688,77 @@ func messageHTMLPlaceholder(m *model.Message) string {
|
||||
return `<a class="` + className + `" href="` + template.HTMLEscapeString(url) + `" target="_blank">` + anchorText + `</a>`
|
||||
})
|
||||
}
|
||||
|
||||
func extractQueryTerms(raw string) []string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
parts := searchTermSplitPattern.Split(raw, -1)
|
||||
terms := make([]string, 0, len(parts))
|
||||
seen := make(map[string]struct{})
|
||||
for _, part := range parts {
|
||||
term := strings.Trim(part, "\"'()[]{}")
|
||||
if term == "" {
|
||||
continue
|
||||
}
|
||||
switch strings.ToUpper(term) {
|
||||
case "AND", "OR", "NOT":
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(term)
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
terms = append(terms, term)
|
||||
}
|
||||
return terms
|
||||
}
|
||||
|
||||
func highlightPlainTextToHTML(text string, terms []string) string {
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
if len(terms) == 0 {
|
||||
return template.HTMLEscapeString(text)
|
||||
}
|
||||
unique := make(map[string]struct{})
|
||||
escaped := make([]string, 0, len(terms))
|
||||
for _, term := range terms {
|
||||
trimmed := strings.TrimSpace(term)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(trimmed)
|
||||
if _, ok := unique[key]; ok {
|
||||
continue
|
||||
}
|
||||
unique[key] = struct{}{}
|
||||
escaped = append(escaped, regexp.QuoteMeta(trimmed))
|
||||
}
|
||||
if len(escaped) == 0 {
|
||||
return template.HTMLEscapeString(text)
|
||||
}
|
||||
sort.Slice(escaped, func(i, j int) bool { return len(escaped[i]) > len(escaped[j]) })
|
||||
pattern := strings.Join(escaped, "|")
|
||||
re, err := regexp.Compile("(?i)" + pattern)
|
||||
if err != nil {
|
||||
return template.HTMLEscapeString(text)
|
||||
}
|
||||
matches := re.FindAllStringIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return template.HTMLEscapeString(text)
|
||||
}
|
||||
var builder strings.Builder
|
||||
last := 0
|
||||
for _, loc := range matches {
|
||||
builder.WriteString(template.HTMLEscapeString(text[last:loc[0]]))
|
||||
builder.WriteString(`<mark class="search-highlight">`)
|
||||
builder.WriteString(template.HTMLEscapeString(text[loc[0]:loc[1]]))
|
||||
builder.WriteString("</mark>")
|
||||
last = loc[1]
|
||||
}
|
||||
builder.WriteString(template.HTMLEscapeString(text[last:]))
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
@@ -190,6 +190,15 @@
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.result-block {
|
||||
margin: 0;
|
||||
white-space: pre-wrap;
|
||||
font-family: "SFMono-Regular", Consolas, "Liberation Mono",
|
||||
Menlo, monospace;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.request-url {
|
||||
background-color: #f0f0f0;
|
||||
padding: 10px;
|
||||
@@ -345,6 +354,13 @@
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.search-highlight {
|
||||
background-color: #fff3b0;
|
||||
color: inherit;
|
||||
padding: 0 2px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.required-field {
|
||||
color: var(--error-color);
|
||||
font-weight: bold;
|
||||
@@ -636,24 +652,27 @@
|
||||
>必填</span
|
||||
></label
|
||||
>
|
||||
<input id="search-query" type="text"
|
||||
placeholder="支持空格分词，利用 AND/OR/\"
|
||||
组合高级语法" />
|
||||
<input
|
||||
id="search-query"
|
||||
type="text"
|
||||
placeholder="支持空格分词，可使用 AND/OR/NOT 等语法"
|
||||
/>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="search-talker"
|
||||
>会话 ID（可多个，英文逗号分隔）：<span
|
||||
class="required-field"
|
||||
>必填</span
|
||||
class="optional-param"
|
||||
>可选</span
|
||||
></label
|
||||
>
|
||||
<input
|
||||
id="search-talker"
|
||||
type="text"
|
||||
placeholder="例如：wxid_xxx@chatroom"
|
||||
placeholder="留空默认搜索所有联系人与群聊"
|
||||
/>
|
||||
<div class="form-hint">
|
||||
可以从“最近会话”或“群聊”接口获取对应的 ID。
|
||||
可以从“最近会话”或“群聊”接口获取对应的
|
||||
ID；如留空将检索所有会话。
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
@@ -669,18 +688,18 @@
|
||||
/>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="search-time"
|
||||
<label for="search-start-date"
|
||||
>时间范围：<span class="optional-param"
|
||||
>可选</span
|
||||
></label
|
||||
>
|
||||
<input
|
||||
id="search-time"
|
||||
type="text"
|
||||
placeholder="支持 2024-01-01~2024-01-31、last-30d 等格式"
|
||||
/>
|
||||
<div class="date-range">
|
||||
<input type="date" id="search-start-date" />
|
||||
<span class="date-separator">至</span>
|
||||
<input type="date" id="search-end-date" />
|
||||
</div>
|
||||
<div class="form-hint">
|
||||
留空表示搜索全量消息；支持与聊天记录接口相同的时间表达式。
|
||||
可选择开始日期和结束日期；留空表示不限制时间范围。
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
@@ -710,6 +729,20 @@
|
||||
placeholder="默认 0"
|
||||
/>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="search-format"
|
||||
>输出格式：<span class="optional-param"
|
||||
>可选</span
|
||||
></label
|
||||
>
|
||||
<select id="search-format">
|
||||
<option value="">默认</option>
|
||||
<option value="json">JSON</option>
|
||||
<option value="text">纯文本</option>
|
||||
<option value="html">HTML</option>
|
||||
<option value="csv">CSV</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button id="test-api">执行查询</button>
|
||||
@@ -833,28 +866,32 @@
|
||||
errorMessage.style.display = "none";
|
||||
errorMessage.textContent = "";
|
||||
|
||||
let highlightTerms = [];
|
||||
let responseFormat = "";
|
||||
|
||||
try {
|
||||
// 获取当前活动的标签
|
||||
const activeTab = document
|
||||
.querySelector(".tab.active")
|
||||
.getAttribute("data-tab");
|
||||
let url = "/api/v1/";
|
||||
let params = new URLSearchParams();
|
||||
const params = new URLSearchParams();
|
||||
|
||||
// 根据不同的标签构建不同的请求
|
||||
switch (activeTab) {
|
||||
case "chatlog":
|
||||
case "chatlog": {
|
||||
url += "chatlog";
|
||||
const startDate =
|
||||
document.getElementById("start-date").value;
|
||||
const endDate =
|
||||
document.getElementById("end-date").value;
|
||||
const talker =
|
||||
document.getElementById("talker").value;
|
||||
const sender =
|
||||
document.getElementById("sender").value;
|
||||
const keyword =
|
||||
document.getElementById("keyword").value;
|
||||
const talker = document
|
||||
.getElementById("talker")
|
||||
.value.trim();
|
||||
const sender = document
|
||||
.getElementById("sender")
|
||||
.value.trim();
|
||||
const keyword = document
|
||||
.getElementById("keyword")
|
||||
.value.trim();
|
||||
const limit =
|
||||
document.getElementById("limit").value;
|
||||
const offset =
|
||||
@@ -862,7 +899,6 @@
|
||||
const format =
|
||||
document.getElementById("format").value;
|
||||
|
||||
// 验证必填项：开始日期
|
||||
if (!startDate && !endDate) {
|
||||
errorMessage.textContent =
|
||||
"错误: 请至少选择开始日期！";
|
||||
@@ -878,7 +914,6 @@
|
||||
}
|
||||
|
||||
let timeValue = "";
|
||||
|
||||
if (startDate && endDate) {
|
||||
if (
|
||||
new Date(startDate) > new Date(endDate)
|
||||
@@ -901,70 +936,85 @@
|
||||
}
|
||||
|
||||
params.append("time", timeValue);
|
||||
// talker 可为空，留空表示全部按会话分组
|
||||
if (talker) params.append("talker", talker);
|
||||
if (sender) params.append("sender", sender);
|
||||
if (keyword) params.append("keyword", keyword);
|
||||
if (keyword) {
|
||||
params.append("keyword", keyword);
|
||||
highlightTerms =
|
||||
extractSearchTerms(keyword);
|
||||
}
|
||||
if (limit) params.append("limit", limit);
|
||||
if (offset) params.append("offset", offset);
|
||||
if (format) params.append("format", format);
|
||||
if (format) {
|
||||
params.append("format", format);
|
||||
responseFormat = format;
|
||||
}
|
||||
break;
|
||||
|
||||
case "contact":
|
||||
}
|
||||
case "contact": {
|
||||
url += "contact";
|
||||
const contactKeyword =
|
||||
document.getElementById(
|
||||
"contact-keyword"
|
||||
).value;
|
||||
const contactKeyword = document
|
||||
.getElementById("contact-keyword")
|
||||
.value.trim();
|
||||
const contactFormat =
|
||||
document.getElementById(
|
||||
"contact-format"
|
||||
).value;
|
||||
|
||||
if (contactKeyword)
|
||||
if (contactKeyword) {
|
||||
params.append("keyword", contactKeyword);
|
||||
if (contactFormat)
|
||||
highlightTerms =
|
||||
extractSearchTerms(contactKeyword);
|
||||
}
|
||||
if (contactFormat) {
|
||||
params.append("format", contactFormat);
|
||||
responseFormat = contactFormat;
|
||||
}
|
||||
break;
|
||||
|
||||
case "chatroom":
|
||||
}
|
||||
case "chatroom": {
|
||||
url += "chatroom";
|
||||
const chatroomKeyword =
|
||||
document.getElementById(
|
||||
"chatroom-keyword"
|
||||
).value;
|
||||
const chatroomKeyword = document
|
||||
.getElementById("chatroom-keyword")
|
||||
.value.trim();
|
||||
const chatroomFormat =
|
||||
document.getElementById(
|
||||
"chatroom-format"
|
||||
).value;
|
||||
|
||||
if (chatroomKeyword)
|
||||
if (chatroomKeyword) {
|
||||
params.append("keyword", chatroomKeyword);
|
||||
if (chatroomFormat)
|
||||
highlightTerms =
|
||||
extractSearchTerms(chatroomKeyword);
|
||||
}
|
||||
if (chatroomFormat) {
|
||||
params.append("format", chatroomFormat);
|
||||
responseFormat = chatroomFormat;
|
||||
}
|
||||
break;
|
||||
|
||||
case "session":
|
||||
}
|
||||
case "session": {
|
||||
url += "session";
|
||||
const sessionFormat =
|
||||
document.getElementById(
|
||||
"session-format"
|
||||
).value;
|
||||
|
||||
if (sessionFormat)
|
||||
if (sessionFormat) {
|
||||
params.append("format", sessionFormat);
|
||||
responseFormat = sessionFormat;
|
||||
}
|
||||
break;
|
||||
case "diary":
|
||||
}
|
||||
case "diary": {
|
||||
url += "diary";
|
||||
const diaryDateInput =
|
||||
document.getElementById("diary-date");
|
||||
const diaryDate = diaryDateInput
|
||||
? diaryDateInput.value
|
||||
: "";
|
||||
const diaryTalker =
|
||||
document.getElementById(
|
||||
"diary-talker"
|
||||
).value;
|
||||
const diaryTalker = document
|
||||
.getElementById("diary-talker")
|
||||
.value.trim();
|
||||
const diaryFormat =
|
||||
document.getElementById(
|
||||
"diary-format"
|
||||
@@ -978,12 +1028,18 @@
|
||||
}
|
||||
|
||||
params.append("date", diaryDate);
|
||||
if (diaryTalker)
|
||||
if (diaryTalker) {
|
||||
params.append("talker", diaryTalker);
|
||||
if (diaryFormat)
|
||||
highlightTerms =
|
||||
extractSearchTerms(diaryTalker);
|
||||
}
|
||||
if (diaryFormat) {
|
||||
params.append("format", diaryFormat);
|
||||
responseFormat = diaryFormat;
|
||||
}
|
||||
break;
|
||||
case "search":
|
||||
}
|
||||
case "search": {
|
||||
url += "search";
|
||||
const searchQuery =
|
||||
document.getElementById(
|
||||
@@ -997,9 +1053,13 @@
|
||||
document.getElementById(
|
||||
"search-sender"
|
||||
).value;
|
||||
const searchTime =
|
||||
const searchStartDate =
|
||||
document.getElementById(
|
||||
"search-time"
|
||||
"search-start-date"
|
||||
).value;
|
||||
const searchEndDate =
|
||||
document.getElementById(
|
||||
"search-end-date"
|
||||
).value;
|
||||
const searchLimit =
|
||||
document.getElementById(
|
||||
@@ -1009,6 +1069,11 @@
|
||||
document.getElementById(
|
||||
"search-offset"
|
||||
).value;
|
||||
const searchFormatSelect =
|
||||
document.getElementById("search-format");
|
||||
const searchFormatValue = searchFormatSelect
|
||||
? searchFormatSelect.value
|
||||
: "";
|
||||
|
||||
if (!searchQuery.trim()) {
|
||||
errorMessage.textContent =
|
||||
@@ -1016,74 +1081,101 @@
|
||||
errorMessage.style.display = "block";
|
||||
return;
|
||||
}
|
||||
if (!searchTalker.trim()) {
|
||||
|
||||
const trimmedQuery = searchQuery.trim();
|
||||
highlightTerms =
|
||||
extractSearchTerms(trimmedQuery);
|
||||
params.append("q", trimmedQuery);
|
||||
|
||||
const trimmedTalker = searchTalker.trim();
|
||||
if (trimmedTalker) {
|
||||
params.append("talker", trimmedTalker);
|
||||
}
|
||||
|
||||
let searchTimeValue = "";
|
||||
if (searchStartDate && searchEndDate) {
|
||||
if (
|
||||
new Date(searchStartDate) >
|
||||
new Date(searchEndDate)
|
||||
) {
|
||||
errorMessage.textContent =
|
||||
"错误: 结束日期不能早于开始日期！";
|
||||
errorMessage.style.display = "block";
|
||||
return;
|
||||
}
|
||||
searchTimeValue = `${searchStartDate}~${searchEndDate}`;
|
||||
} else if (searchStartDate) {
|
||||
searchTimeValue = searchStartDate;
|
||||
} else if (!searchStartDate && searchEndDate) {
|
||||
errorMessage.textContent =
|
||||
"错误: 请输入会话 ID！";
|
||||
"错误: 请先选择开始日期！";
|
||||
errorMessage.style.display = "block";
|
||||
return;
|
||||
}
|
||||
|
||||
params.append("q", searchQuery.trim());
|
||||
params.append("talker", searchTalker.trim());
|
||||
if (searchSender)
|
||||
params.append(
|
||||
"sender",
|
||||
searchSender.trim()
|
||||
);
|
||||
if (searchTime)
|
||||
params.append("time", searchTime.trim());
|
||||
if (searchTimeValue)
|
||||
params.append("time", searchTimeValue);
|
||||
if (searchLimit)
|
||||
params.append("limit", searchLimit);
|
||||
if (searchOffset)
|
||||
params.append("offset", searchOffset);
|
||||
if (searchFormatValue) {
|
||||
params.append("format", searchFormatValue);
|
||||
responseFormat = searchFormatValue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 添加参数到URL
|
||||
const apiUrl = params.toString()
|
||||
? `${url}?${params.toString()}`
|
||||
: url;
|
||||
|
||||
// 获取完整URL（包含域名部分）
|
||||
const fullUrl = window.location.origin + apiUrl;
|
||||
|
||||
// 显示完整请求URL
|
||||
requestUrlContainer.textContent = fullUrl;
|
||||
resultWrapper.style.display = "block";
|
||||
|
||||
// 显示加载中
|
||||
resultContainer.innerHTML =
|
||||
'<div class="loading">加载中</div>';
|
||||
|
||||
// 发送请求
|
||||
const response = await fetch(apiUrl);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`HTTP error! Status: ${response.status}`
|
||||
);
|
||||
}
|
||||
|
||||
// 获取响应内容类型
|
||||
const contentType =
|
||||
response.headers.get("content-type");
|
||||
let result;
|
||||
|
||||
if (
|
||||
contentType &&
|
||||
contentType.includes("application/json")
|
||||
) {
|
||||
// 如果是JSON，格式化显示
|
||||
result = await response.json();
|
||||
resultContainer.innerHTML = JSON.stringify(
|
||||
result,
|
||||
null,
|
||||
2
|
||||
response.headers.get("content-type") || "";
|
||||
if (contentType.includes("application/json")) {
|
||||
const data = await response.json();
|
||||
const text = JSON.stringify(data, null, 2);
|
||||
const highlighted = highlightPlainText(
|
||||
text,
|
||||
highlightTerms
|
||||
);
|
||||
resultContainer.innerHTML = `<pre class="result-block">${highlighted}</pre>`;
|
||||
} else {
|
||||
// 其他格式直接显示文本
|
||||
result = await response.text();
|
||||
resultContainer.innerHTML = result;
|
||||
const textResult = await response.text();
|
||||
const expectsHtml =
|
||||
(responseFormat || "").toLowerCase() ===
|
||||
"html" || contentType.includes("text/html");
|
||||
if (expectsHtml) {
|
||||
resultContainer.innerHTML = textResult;
|
||||
highlightHtmlContent(
|
||||
resultContainer,
|
||||
highlightTerms
|
||||
);
|
||||
} else {
|
||||
const highlighted = highlightPlainText(
|
||||
textResult,
|
||||
highlightTerms
|
||||
);
|
||||
resultContainer.innerHTML = `<pre class="result-block">${highlighted}</pre>`;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
resultContainer.innerHTML = "";
|
||||
@@ -1093,6 +1185,112 @@
|
||||
}
|
||||
});
|
||||
|
||||
function escapeHtml(str) {
|
||||
if (str == null) return "";
|
||||
return str
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
function extractSearchTerms(raw) {
|
||||
if (!raw) return [];
|
||||
return Array.from(
|
||||
new Set(
|
||||
raw
|
||||
.split(/[\s,，；;、]+/)
|
||||
.map((term) => term.trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function buildHighlightPattern(terms) {
|
||||
if (!terms || !terms.length) return "";
|
||||
const escaped = Array.from(
|
||||
new Set(
|
||||
terms
|
||||
.map((term) =>
|
||||
term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
|
||||
)
|
||||
.filter(Boolean)
|
||||
)
|
||||
).sort((a, b) => b.length - a.length);
|
||||
if (!escaped.length) return "";
|
||||
return `(${escaped.join("|")})`;
|
||||
}
|
||||
|
||||
function highlightPlainText(text, terms) {
|
||||
const pattern = buildHighlightPattern(terms);
|
||||
if (!pattern) {
|
||||
return escapeHtml(text);
|
||||
}
|
||||
const regex = new RegExp(pattern, "gi");
|
||||
let lastIndex = 0;
|
||||
let highlighted = "";
|
||||
text.replace(regex, (match, _group, offset) => {
|
||||
highlighted += escapeHtml(text.slice(lastIndex, offset));
|
||||
highlighted += `<mark class="search-highlight">${escapeHtml(
|
||||
match
|
||||
)}</mark>`;
|
||||
lastIndex = offset + match.length;
|
||||
return match;
|
||||
});
|
||||
highlighted += escapeHtml(text.slice(lastIndex));
|
||||
return highlighted;
|
||||
}
|
||||
|
||||
function highlightHtmlContent(container, terms) {
|
||||
const pattern = buildHighlightPattern(terms);
|
||||
if (!pattern) return;
|
||||
const walker = document.createTreeWalker(
|
||||
container,
|
||||
NodeFilter.SHOW_TEXT,
|
||||
null,
|
||||
false
|
||||
);
|
||||
const nodes = [];
|
||||
while (walker.nextNode()) {
|
||||
const node = walker.currentNode;
|
||||
if (!node || !node.nodeValue || !node.nodeValue.trim())
|
||||
continue;
|
||||
nodes.push(node);
|
||||
}
|
||||
nodes.forEach((textNode) => {
|
||||
const text = textNode.nodeValue;
|
||||
const regex = new RegExp(pattern, "gi");
|
||||
if (!regex.test(text)) {
|
||||
return;
|
||||
}
|
||||
regex.lastIndex = 0;
|
||||
const fragment = document.createDocumentFragment();
|
||||
let lastIndex = 0;
|
||||
text.replace(regex, (match, _group, offset) => {
|
||||
if (offset > lastIndex) {
|
||||
fragment.appendChild(
|
||||
document.createTextNode(
|
||||
text.slice(lastIndex, offset)
|
||||
)
|
||||
);
|
||||
}
|
||||
const mark = document.createElement("mark");
|
||||
mark.className = "search-highlight";
|
||||
mark.textContent = match;
|
||||
fragment.appendChild(mark);
|
||||
lastIndex = offset + match.length;
|
||||
return match;
|
||||
});
|
||||
if (lastIndex < text.length) {
|
||||
fragment.appendChild(
|
||||
document.createTextNode(text.slice(lastIndex))
|
||||
);
|
||||
}
|
||||
textNode.parentNode.replaceChild(fragment, textNode);
|
||||
});
|
||||
}
|
||||
|
||||
const diaryDateInput = document.getElementById("diary-date");
|
||||
if (diaryDateInput) {
|
||||
const today = new Date();
|
||||
|
||||
@@ -98,8 +98,7 @@ func NewGroup(ctx context.Context, group string, hooks []Webhook, delayMs int64)
|
||||
}
|
||||
|
||||
func (g *Group) Callback(event fsnotify.Event) error {
|
||||
// skip remove event
|
||||
if !event.Op.Has(fsnotify.Create) {
|
||||
if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+22
-11
@@ -6,7 +6,7 @@ import "time"
|
||||
// Start 和 End 为闭区间；如未提供则由调用方决定默认范围
|
||||
// Limit/Offset 由调用链路在进入数据源前进行裁剪
|
||||
// Sender 使用英文逗号分隔多个筛选条件
|
||||
// Talker 目前要求调用方传入具体会话（可多选，英文逗号分隔）
|
||||
// Talker 可选：留空时后端会遍历所有会话；如需限定多个会话，使用英文逗号分隔
|
||||
type SearchRequest struct {
|
||||
Query string `json:"query"`
|
||||
Talker string `json:"talker"`
|
||||
@@ -39,14 +39,25 @@ type SearchHit struct {
|
||||
// Limit / Offset 为实际生效的分页参数
|
||||
// Hits 序列按相关度排序，命中数可能小于 limit（例如过滤后不足）
|
||||
type SearchResponse struct {
|
||||
Total int `json:"total"`
|
||||
Hits []*SearchHit `json:"hits"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
Query string `json:"query"`
|
||||
Talker string `json:"talker"`
|
||||
Sender string `json:"sender"`
|
||||
Start time.Time `json:"start"`
|
||||
End time.Time `json:"end"`
|
||||
Total int `json:"total"`
|
||||
Hits []*SearchHit `json:"hits"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
Query string `json:"query"`
|
||||
Talker string `json:"talker"`
|
||||
Sender string `json:"sender"`
|
||||
Start time.Time `json:"start"`
|
||||
End time.Time `json:"end"`
|
||||
Index *SearchIndexStatus `json:"index_status,omitempty"`
|
||||
}
|
||||
|
||||
// SearchIndexStatus 表示 Bleve 索引的构建状态
|
||||
type SearchIndexStatus struct {
|
||||
Ready bool `json:"ready"`
|
||||
InProgress bool `json:"in_progress"`
|
||||
Progress float64 `json:"progress"`
|
||||
LastStartedAt time.Time `json:"last_started_at"`
|
||||
LastCompletedAt time.Time `json:"last_completed_at"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
@@ -819,6 +819,10 @@ func (ds *DataSource) Close() error {
|
||||
return ds.dbm.Close()
|
||||
}
|
||||
|
||||
func (ds *DataSource) GetDatasetFingerprint(context.Context) (string, error) {
|
||||
return ds.dbm.FingerprintForGroups(Message)
|
||||
}
|
||||
|
||||
// GetAvatar returns not found for darwin v3 (no head image source known here)
|
||||
func (ds *DataSource) GetAvatar(ctx context.Context, username string, size string) (*model.Avatar, error) {
|
||||
return nil, errors.ErrAvatarNotFound
|
||||
|
||||
@@ -18,6 +18,7 @@ type DataSource interface {
|
||||
// 消息
|
||||
GetMessages(ctx context.Context, startTime, endTime time.Time, talker string, sender string, keyword string, limit, offset int) ([]*model.Message, error)
|
||||
SearchMessages(ctx context.Context, req *model.SearchRequest) (*model.SearchResponse, error)
|
||||
GetDatasetFingerprint(ctx context.Context) (string, error)
|
||||
|
||||
// 联系人
|
||||
GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error)
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package dbm
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// FingerprintForGroups calculates a stable fingerprint for the provided file groups.
|
||||
// It inspects the current file list managed by DBManager, collects their size and
|
||||
// modification timestamp, and hashes the aggregated metadata. If none of the
|
||||
// requested groups contain files, an empty string is returned without error.
|
||||
func (d *DBManager) FingerprintForGroups(names ...string) (string, error) {
|
||||
if len(names) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
type fileFingerprint struct {
|
||||
group string
|
||||
rel string
|
||||
size int64
|
||||
mod int64
|
||||
}
|
||||
|
||||
entries := make([]fileFingerprint, 0)
|
||||
|
||||
for _, name := range names {
|
||||
paths, err := d.GetDBPath(name)
|
||||
if err != nil {
|
||||
// When the underlying error indicates the files are simply missing,
|
||||
// treat it as an empty group rather than a hard failure.
|
||||
if strings.Contains(err.Error(), "db file not found") {
|
||||
continue
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
info, statErr := os.Stat(p)
|
||||
if statErr != nil {
|
||||
if os.IsNotExist(statErr) {
|
||||
continue
|
||||
}
|
||||
return "", statErr
|
||||
}
|
||||
|
||||
rel := p
|
||||
if relPath, relErr := filepath.Rel(d.path, p); relErr == nil {
|
||||
rel = relPath
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
|
||||
entries = append(entries, fileFingerprint{
|
||||
group: name,
|
||||
rel: rel,
|
||||
size: info.Size(),
|
||||
mod: info.ModTime().UnixNano(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
if entries[i].group == entries[j].group {
|
||||
return entries[i].rel < entries[j].rel
|
||||
}
|
||||
return entries[i].group < entries[j].group
|
||||
})
|
||||
|
||||
hasher := sha256.New()
|
||||
for _, e := range entries {
|
||||
fmt.Fprintf(hasher, "%s|%s|%d|%d;", e.group, e.rel, e.size, e.mod)
|
||||
}
|
||||
|
||||
return hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
@@ -644,6 +644,10 @@ func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchReque
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ds *DataSource) GetDatasetFingerprint(context.Context) (string, error) {
|
||||
return ds.dbm.FingerprintForGroups(Message)
|
||||
}
|
||||
|
||||
// 联系人
|
||||
func (ds *DataSource) GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error) {
|
||||
var query string
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
@@ -28,6 +29,8 @@ const (
|
||||
Video = "video"
|
||||
File = "file"
|
||||
Voice = "voice"
|
||||
|
||||
talkerCacheTTL = 30 * time.Second
|
||||
)
|
||||
|
||||
var Groups = []*dbm.Group{
|
||||
@@ -78,6 +81,10 @@ type DataSource struct {
|
||||
|
||||
// 消息数据库信息
|
||||
messageInfos []MessageDBInfo
|
||||
|
||||
talkerCacheMu sync.RWMutex
|
||||
talkerCache []string
|
||||
talkerCacheExpiry time.Time
|
||||
}
|
||||
|
||||
// New 创建一个新的 WindowsV3DataSource
|
||||
@@ -101,12 +108,21 @@ func New(path string) (*DataSource, error) {
|
||||
}
|
||||
|
||||
ds.dbm.AddCallback(Message, func(event fsnotify.Event) error {
|
||||
if !event.Op.Has(fsnotify.Create) {
|
||||
if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename)) {
|
||||
return nil
|
||||
}
|
||||
if err := ds.initMessageDbs(); err != nil {
|
||||
log.Err(err).Msgf("Failed to reinitialize message DBs: %s", event.Name)
|
||||
}
|
||||
ds.invalidateTalkerCache()
|
||||
return nil
|
||||
})
|
||||
|
||||
ds.dbm.AddCallback(Contact, func(event fsnotify.Event) error {
|
||||
if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename) || event.Op.Has(fsnotify.Remove)) {
|
||||
return nil
|
||||
}
|
||||
ds.invalidateTalkerCache()
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -214,6 +230,7 @@ func (ds *DataSource) initMessageDbs() error {
|
||||
return nil
|
||||
}
|
||||
ds.messageInfos = infos
|
||||
ds.invalidateTalkerCache()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -429,7 +446,25 @@ func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchReque
|
||||
|
||||
talkers := util.Str2List(req.Talker, ",")
|
||||
if len(talkers) == 0 {
|
||||
return nil, errors.ErrTalkerEmpty
|
||||
var err error
|
||||
talkers, err = ds.collectAllTalkers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(talkers) == 0 {
|
||||
return &model.SearchResponse{
|
||||
Total: 0,
|
||||
Hits: []*model.SearchHit{},
|
||||
DurationMs: 0,
|
||||
Limit: req.Limit,
|
||||
Offset: req.Offset,
|
||||
Query: req.Query,
|
||||
Talker: req.Talker,
|
||||
Sender: req.Sender,
|
||||
Start: req.Start,
|
||||
End: req.End,
|
||||
}, nil
|
||||
}
|
||||
|
||||
limit := req.Limit
|
||||
@@ -676,6 +711,117 @@ func (ds *DataSource) SearchMessages(ctx context.Context, req *model.SearchReque
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (ds *DataSource) GetDatasetFingerprint(context.Context) (string, error) {
|
||||
return ds.dbm.FingerprintForGroups(Message)
|
||||
}
|
||||
|
||||
func (ds *DataSource) getCachedTalkers() []string {
|
||||
ds.talkerCacheMu.RLock()
|
||||
if ds.talkerCacheExpiry.IsZero() || time.Now().After(ds.talkerCacheExpiry) {
|
||||
ds.talkerCacheMu.RUnlock()
|
||||
return nil
|
||||
}
|
||||
cached := append([]string(nil), ds.talkerCache...)
|
||||
ds.talkerCacheMu.RUnlock()
|
||||
return cached
|
||||
}
|
||||
|
||||
func (ds *DataSource) cacheTalkers(talkers []string) {
|
||||
copySlice := append([]string(nil), talkers...)
|
||||
ds.talkerCacheMu.Lock()
|
||||
ds.talkerCache = copySlice
|
||||
ds.talkerCacheExpiry = time.Now().Add(talkerCacheTTL)
|
||||
ds.talkerCacheMu.Unlock()
|
||||
}
|
||||
|
||||
func (ds *DataSource) invalidateTalkerCache() {
|
||||
ds.talkerCacheMu.Lock()
|
||||
ds.talkerCache = nil
|
||||
ds.talkerCacheExpiry = time.Time{}
|
||||
ds.talkerCacheMu.Unlock()
|
||||
}
|
||||
|
||||
func (ds *DataSource) collectAllTalkers(ctx context.Context) ([]string, error) {
|
||||
if cached := ds.getCachedTalkers(); cached != nil {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
talkerSet := make(map[string]struct{})
|
||||
talkers := make([]string, 0)
|
||||
|
||||
add := func(name string) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
if _, exists := talkerSet[name]; exists {
|
||||
return
|
||||
}
|
||||
talkerSet[name] = struct{}{}
|
||||
talkers = append(talkers, name)
|
||||
}
|
||||
|
||||
if db, err := ds.dbm.GetDB(Contact); err == nil && db != nil {
|
||||
rows, err := db.QueryContext(ctx, `SELECT strUsrName FROM Session WHERE IFNULL(strUsrName,'') <> ''`)
|
||||
if err == nil {
|
||||
for rows.Next() {
|
||||
if err := ctx.Err(); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
var username string
|
||||
if err := rows.Scan(&username); err != nil {
|
||||
log.Debug().Err(err).Msg("scan session talker failed")
|
||||
continue
|
||||
}
|
||||
add(username)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Debug().Err(err).Msg("iterate session talkers failed")
|
||||
}
|
||||
rows.Close()
|
||||
} else {
|
||||
log.Debug().Err(err).Msg("query session talkers failed")
|
||||
}
|
||||
|
||||
if len(talkers) == 0 {
|
||||
rows, err = db.QueryContext(ctx, `SELECT UserName FROM Contact WHERE IFNULL(UserName,'') <> ''`)
|
||||
if err == nil {
|
||||
for rows.Next() {
|
||||
if err := ctx.Err(); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
var username string
|
||||
if err := rows.Scan(&username); err != nil {
|
||||
log.Debug().Err(err).Msg("scan contact talker failed")
|
||||
continue
|
||||
}
|
||||
add(username)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Debug().Err(err).Msg("iterate contact talkers failed")
|
||||
}
|
||||
rows.Close()
|
||||
} else {
|
||||
log.Debug().Err(err).Msg("query contact talkers failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(talkers) == 0 {
|
||||
for _, info := range ds.messageInfos {
|
||||
for username := range info.TalkerMap {
|
||||
add(username)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(talkers)
|
||||
ds.cacheTalkers(talkers)
|
||||
return talkers, nil
|
||||
}
|
||||
|
||||
// GetContacts 实现获取联系人信息的方法
|
||||
func (ds *DataSource) GetContacts(ctx context.Context, key string, limit, offset int) ([]*model.Contact, error) {
|
||||
var query string
|
||||
|
||||
@@ -0,0 +1,482 @@
|
||||
package indexer
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/mapping"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
|
||||
"github.com/sjzar/chatlog/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
runtimeIndexVersionKey = "chatlog_index_version"
|
||||
runtimeIndexVersion = "1"
|
||||
fingerprintKey = "chatlog_index_fingerprint"
|
||||
lastBuiltKey = "chatlog_index_last_built"
|
||||
)
|
||||
|
||||
// SearchHit represents a single Bleve search hit mapped back to chatlog's domain model.
|
||||
type SearchHit struct {
|
||||
Message *model.Message
|
||||
Snippet string
|
||||
Score float64
|
||||
}
|
||||
|
||||
// Index wraps a Bleve index with concurrency control and helper utilities.
|
||||
type Index struct {
|
||||
mu sync.RWMutex
|
||||
idx bleve.Index
|
||||
path string
|
||||
}
|
||||
|
||||
// Open opens an existing index at the given path or creates a new one with the default mapping.
|
||||
func Open(path string) (*Index, error) {
|
||||
parent := filepath.Dir(path)
|
||||
if err := os.MkdirAll(parent, 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create index parent dir: %w", err)
|
||||
}
|
||||
|
||||
var (
|
||||
idx bleve.Index
|
||||
err error
|
||||
)
|
||||
if _, statErr := os.Stat(path); statErr == nil {
|
||||
idx, err = bleve.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open bleve index: %w", err)
|
||||
}
|
||||
} else if errors.Is(statErr, os.ErrNotExist) {
|
||||
mapping, mapErr := buildMapping()
|
||||
if mapErr != nil {
|
||||
return nil, mapErr
|
||||
}
|
||||
idx, err = bleve.New(path, mapping)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create bleve index: %w", err)
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("stat index: %w", statErr)
|
||||
}
|
||||
|
||||
return &Index{idx: idx, path: path}, nil
|
||||
}
|
||||
|
||||
// Close closes the underlying Bleve index.
|
||||
func (i *Index) Close() error {
|
||||
if i == nil {
|
||||
return nil
|
||||
}
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
if i.idx == nil {
|
||||
return nil
|
||||
}
|
||||
return i.idx.Close()
|
||||
}
|
||||
|
||||
// Reset drops all existing data and recreates the index with the default mapping.
|
||||
func (i *Index) Reset() error {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
if i.idx != nil {
|
||||
_ = i.idx.Close()
|
||||
}
|
||||
if err := os.RemoveAll(i.path); err != nil {
|
||||
return fmt.Errorf("remove index dir: %w", err)
|
||||
}
|
||||
mapping, err := buildMapping()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
idx, err := bleve.New(i.path, mapping)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recreate bleve index: %w", err)
|
||||
}
|
||||
i.idx = idx
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMetadata stores an internal key/value pair within the index (not searchable).
|
||||
func (i *Index) SetMetadata(key string, value []byte) error {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
if i.idx == nil {
|
||||
return errors.New("index not initialized")
|
||||
}
|
||||
return i.idx.SetInternal([]byte(key), value)
|
||||
}
|
||||
|
||||
// GetMetadata retrieves an internal key/value pair.
|
||||
func (i *Index) GetMetadata(key string) ([]byte, error) {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
if i.idx == nil {
|
||||
return nil, errors.New("index not initialized")
|
||||
}
|
||||
return i.idx.GetInternal([]byte(key))
|
||||
}
|
||||
|
||||
// IndexMessages indexes a batch of messages using Bleve batches for efficiency.
|
||||
func (i *Index) IndexMessages(messages []*model.Message) error {
|
||||
if len(messages) == 0 {
|
||||
return nil
|
||||
}
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
if i.idx == nil {
|
||||
return errors.New("index not initialized")
|
||||
}
|
||||
|
||||
batch := i.idx.NewBatch()
|
||||
const batchSize = 250
|
||||
for idx, msg := range messages {
|
||||
doc, err := newDocument(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := batch.Index(doc.ID, doc); err != nil {
|
||||
return fmt.Errorf("batch index: %w", err)
|
||||
}
|
||||
if (idx+1)%batchSize == 0 {
|
||||
if err := i.idx.Batch(batch); err != nil {
|
||||
return fmt.Errorf("flush batch: %w", err)
|
||||
}
|
||||
batch = i.idx.NewBatch()
|
||||
}
|
||||
}
|
||||
|
||||
if batch.Size() > 0 {
|
||||
if err := i.idx.Batch(batch); err != nil {
|
||||
return fmt.Errorf("flush final batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Search executes a Bleve search over the indexed content applying optional filters.
|
||||
func (i *Index) Search(req *model.SearchRequest, talkers []string, senders []string, startUnix, endUnix int64, offset, limit int) ([]*SearchHit, int, error) {
|
||||
if req == nil {
|
||||
return nil, 0, errors.New("search request is nil")
|
||||
}
|
||||
|
||||
queryObj := buildQuery(req.Query, talkers, senders, startUnix, endUnix)
|
||||
if queryObj == nil {
|
||||
return []*SearchHit{}, 0, nil
|
||||
}
|
||||
|
||||
i.mu.RLock()
|
||||
idx := i.idx
|
||||
i.mu.RUnlock()
|
||||
if idx == nil {
|
||||
return nil, 0, errors.New("index not initialized")
|
||||
}
|
||||
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if limit > 200 {
|
||||
limit = 200
|
||||
}
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
searchRequest := bleve.NewSearchRequestOptions(queryObj, limit, offset, false)
|
||||
searchRequest.Highlight = bleve.NewHighlightWithStyle("html")
|
||||
searchRequest.Fields = []string{"message_json"}
|
||||
searchRequest.IncludeLocations = false
|
||||
|
||||
result, err := idx.Search(searchRequest)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("bleve search: %w", err)
|
||||
}
|
||||
|
||||
hits := make([]*SearchHit, 0, len(result.Hits))
|
||||
for _, hit := range result.Hits {
|
||||
messageJSON, ok := hit.Fields["message_json"].(string)
|
||||
if !ok || messageJSON == "" {
|
||||
continue
|
||||
}
|
||||
var msg model.Message
|
||||
if err := json.Unmarshal([]byte(messageJSON), &msg); err != nil {
|
||||
return nil, 0, fmt.Errorf("decode message: %w", err)
|
||||
}
|
||||
|
||||
snippet := ""
|
||||
if frags, ok := hit.Fragments["content"]; ok && len(frags) > 0 {
|
||||
snippet = strings.Join(frags, " … ")
|
||||
}
|
||||
|
||||
hits = append(hits, &SearchHit{
|
||||
Message: &msg,
|
||||
Snippet: snippet,
|
||||
Score: hit.Score,
|
||||
})
|
||||
}
|
||||
|
||||
return hits, int(result.Total), nil
|
||||
}
|
||||
|
||||
// Document representation stored inside Bleve.
|
||||
type document struct {
|
||||
ID string `json:"id"`
|
||||
Talker string `json:"talker"`
|
||||
Sender string `json:"sender"`
|
||||
Unix int64 `json:"unix"`
|
||||
Seq int64 `json:"seq"`
|
||||
Content string `json:"content"`
|
||||
MessageJSON string `json:"message_json"`
|
||||
}
|
||||
|
||||
func newDocument(msg *model.Message) (*document, error) {
|
||||
if msg == nil {
|
||||
return nil, errors.New("nil message")
|
||||
}
|
||||
content := msg.PlainTextContent()
|
||||
messageJSON, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal message: %w", err)
|
||||
}
|
||||
return &document{
|
||||
ID: fmt.Sprintf("%s:%d", msg.Talker, msg.Seq),
|
||||
Talker: msg.Talker,
|
||||
Sender: msg.Sender,
|
||||
Unix: msg.Time.Unix(),
|
||||
Seq: msg.Seq,
|
||||
Content: content,
|
||||
MessageJSON: string(messageJSON),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildMapping() (*mapping.IndexMappingImpl, error) {
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
indexMapping.TypeField = "type"
|
||||
indexMapping.DefaultAnalyzer = "standard"
|
||||
|
||||
docMapping := mapping.NewDocumentMapping()
|
||||
|
||||
contentField := mapping.NewTextFieldMapping()
|
||||
contentField.Analyzer = "standard"
|
||||
contentField.Store = false
|
||||
docMapping.AddFieldMappingsAt("content", contentField)
|
||||
|
||||
talkerField := mapping.NewTextFieldMapping()
|
||||
talkerField.Analyzer = "keyword"
|
||||
talkerField.Store = true
|
||||
talkerField.IncludeInAll = false
|
||||
docMapping.AddFieldMappingsAt("talker", talkerField)
|
||||
|
||||
senderField := mapping.NewTextFieldMapping()
|
||||
senderField.Analyzer = "keyword"
|
||||
senderField.Store = true
|
||||
senderField.IncludeInAll = false
|
||||
docMapping.AddFieldMappingsAt("sender", senderField)
|
||||
|
||||
unixField := mapping.NewNumericFieldMapping()
|
||||
unixField.Store = true
|
||||
docMapping.AddFieldMappingsAt("unix", unixField)
|
||||
|
||||
seqField := mapping.NewNumericFieldMapping()
|
||||
seqField.Store = true
|
||||
docMapping.AddFieldMappingsAt("seq", seqField)
|
||||
|
||||
messageField := mapping.NewTextFieldMapping()
|
||||
messageField.Analyzer = "keyword"
|
||||
messageField.Store = true
|
||||
messageField.Index = false
|
||||
docMapping.AddFieldMappingsAt("message_json", messageField)
|
||||
|
||||
indexMapping.DefaultMapping = docMapping
|
||||
|
||||
return indexMapping, nil
|
||||
}
|
||||
|
||||
func buildQuery(input string, talkers []string, senders []string, startUnix, endUnix int64) query.Query {
|
||||
contentQuery := buildContentQuery(input)
|
||||
|
||||
var must []query.Query
|
||||
if contentQuery != nil {
|
||||
must = append(must, contentQuery)
|
||||
}
|
||||
|
||||
if len(talkers) > 0 {
|
||||
must = append(must, buildTermsFilter("talker", talkers))
|
||||
}
|
||||
if len(senders) > 0 {
|
||||
must = append(must, buildTermsFilter("sender", senders))
|
||||
}
|
||||
if startUnix > 0 || endUnix > 0 {
|
||||
var minPtr, maxPtr *float64
|
||||
if startUnix > 0 {
|
||||
min := float64(startUnix)
|
||||
minPtr = &min
|
||||
}
|
||||
if endUnix > 0 {
|
||||
max := float64(endUnix)
|
||||
maxPtr = &max
|
||||
}
|
||||
rangeQuery := query.NewNumericRangeQuery(minPtr, maxPtr)
|
||||
rangeQuery.SetField("unix")
|
||||
must = append(must, rangeQuery)
|
||||
}
|
||||
|
||||
if len(must) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(must) == 1 {
|
||||
return must[0]
|
||||
}
|
||||
|
||||
return query.NewConjunctionQuery(must)
|
||||
}
|
||||
|
||||
func buildContentQuery(input string) query.Query {
|
||||
s := strings.TrimSpace(input)
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
upper := strings.ToUpper(s)
|
||||
advanced := strings.ContainsAny(s, "\"'*()") ||
|
||||
strings.Contains(upper, " AND ") ||
|
||||
strings.Contains(upper, " OR ") ||
|
||||
strings.Contains(upper, " NEAR ") ||
|
||||
strings.HasPrefix(upper, "NOT ")
|
||||
|
||||
if advanced {
|
||||
return query.NewQueryStringQuery(s)
|
||||
}
|
||||
|
||||
tokens := strings.Fields(s)
|
||||
if len(tokens) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(tokens) == 1 {
|
||||
mq := query.NewMatchQuery(tokens[0])
|
||||
mq.SetField("content")
|
||||
return mq
|
||||
}
|
||||
|
||||
conj := make([]query.Query, 0, len(tokens))
|
||||
for _, token := range tokens {
|
||||
t := strings.TrimSpace(token)
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
mq := query.NewMatchQuery(t)
|
||||
mq.SetField("content")
|
||||
conj = append(conj, mq)
|
||||
}
|
||||
|
||||
if len(conj) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(conj) == 1 {
|
||||
return conj[0]
|
||||
}
|
||||
return query.NewConjunctionQuery(conj)
|
||||
}
|
||||
|
||||
func buildTermsFilter(field string, values []string) query.Query {
|
||||
sanitized := make([]query.Query, 0, len(values))
|
||||
for _, val := range values {
|
||||
trimmed := strings.TrimSpace(val)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
tq := query.NewTermQuery(trimmed)
|
||||
tq.SetField(field)
|
||||
sanitized = append(sanitized, tq)
|
||||
}
|
||||
if len(sanitized) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(sanitized) == 1 {
|
||||
return sanitized[0]
|
||||
}
|
||||
return query.NewDisjunctionQuery(sanitized)
|
||||
}
|
||||
|
||||
// EnsureVersion ensures the stored index version matches current runtime version.
|
||||
func (i *Index) EnsureVersion() (bool, error) {
|
||||
current, _ := i.GetMetadata(runtimeIndexVersionKey)
|
||||
if string(current) == runtimeIndexVersion {
|
||||
return true, nil
|
||||
}
|
||||
if err := i.SetMetadata(runtimeIndexVersionKey, []byte(runtimeIndexVersion)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// UpdateFingerprint persists the dataset fingerprint into index metadata.
|
||||
func (i *Index) UpdateFingerprint(fp string) error {
|
||||
return i.SetMetadata(fingerprintKey, []byte(fp))
|
||||
}
|
||||
|
||||
// Fingerprint reads the stored dataset fingerprint from index metadata.
|
||||
func (i *Index) Fingerprint() string {
|
||||
b, err := i.GetMetadata(fingerprintKey)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// FingerprintMatches reports whether the stored dataset fingerprint equals the provided value.
|
||||
func (i *Index) FingerprintMatches(fp string) bool {
|
||||
if fp == "" {
|
||||
return false
|
||||
}
|
||||
return i.Fingerprint() == fp
|
||||
}
|
||||
|
||||
// EnsureFingerprint compares the stored fingerprint with the provided value.
|
||||
// It returns true when the fingerprint already matches. When it differs, the
|
||||
// metadata is updated to the new fingerprint and false is returned to signal
|
||||
// callers that a rebuild just occurred.
|
||||
func (i *Index) EnsureFingerprint(fp string) (bool, error) {
|
||||
if fp == "" {
|
||||
return false, nil
|
||||
}
|
||||
current := i.Fingerprint()
|
||||
if current == fp {
|
||||
return true, nil
|
||||
}
|
||||
if err := i.UpdateFingerprint(fp); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// UpdateLastBuilt 记录最近一次索引构建完成时间（Unix 秒）
|
||||
func (i *Index) UpdateLastBuilt(t time.Time) error {
|
||||
return i.SetMetadata(lastBuiltKey, []byte(strconv.FormatInt(t.Unix(), 10)))
|
||||
}
|
||||
|
||||
// LastBuilt 返回最近一次索引构建完成时间
|
||||
func (i *Index) LastBuilt() time.Time {
|
||||
b, err := i.GetMetadata(lastBuiltKey)
|
||||
if err != nil || len(b) == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
sec, parseErr := strconv.ParseInt(string(b), 10, 64)
|
||||
if parseErr != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(sec, 0)
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func (r *Repository) initCache(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (r *Repository) contactCallback(event fsnotify.Event) error {
|
||||
if !event.Op.Has(fsnotify.Create) {
|
||||
if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename) || event.Op.Has(fsnotify.Remove)) {
|
||||
return nil
|
||||
}
|
||||
if err := r.initContactCache(context.Background()); err != nil {
|
||||
@@ -96,7 +96,7 @@ func (r *Repository) contactCallback(event fsnotify.Event) error {
|
||||
}
|
||||
|
||||
func (r *Repository) chatroomCallback(event fsnotify.Event) error {
|
||||
if !event.Op.Has(fsnotify.Create) {
|
||||
if !(event.Op.Has(fsnotify.Create) || event.Op.Has(fsnotify.Write) || event.Op.Has(fsnotify.Rename) || event.Op.Has(fsnotify.Remove)) {
|
||||
return nil
|
||||
}
|
||||
if err := r.initChatRoomCache(context.Background()); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user