Alawn/chatlog2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(fts) : Auto update index using webhook

Browse Source
This commit is contained in:
Symb0x76
2025-10-20 09:25:48 +08:00
parent 38d5a590de
commit cfd16294b5
12 changed files with 448 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/chatlog/http/static/index.htm
+117
View File
@@ -272,6 +272,18 @@
animation: dots 1.5s steps(5, end) infinite;
}
.search-index-status {
display: none;
margin-bottom: 16px;
padding: 12px 14px;
border-radius: 6px;
border: 1px solid #ffe1a3;
background-color: #fff8e5;
color: #8a6d3b;
font-size: 14px;
line-height: 1.5;
}
@keyframes dots {
0%,
20% {
@@ -854,6 +866,14 @@
>
</p>
</div>
<div
id="search-index-status"
class="search-index-status"
role="status"
aria-live="polite"
>
正在检查全文索引状态...
</div>
<div class="form-group">
<label for="search-start-date"
>时间范围：<span class="optional-param"
@@ -1681,6 +1701,103 @@
document.getElementById("error-message");
const copyResultButton = document.getElementById("copy-result");
const copyUrlButton = document.getElementById("copy-url");
const searchIndexStatusBanner = document.getElementById(
"search-index-status"
);
let searchIndexReady = false;
let searchIndexStatusTimer = null;
if (searchIndexStatusBanner) {
searchIndexStatusBanner.dataset.statusText =
searchIndexStatusBanner.textContent.trim();
}
function updateSearchIndexBanner(status, options) {
if (!searchIndexStatusBanner) {
searchIndexReady = Boolean(status && status.ready);
return;
}
if (options && options.error) {
searchIndexReady = false;
searchIndexStatusBanner.style.display = "block";
searchIndexStatusBanner.textContent = options.error;
searchIndexStatusBanner.dataset.statusText = options.error;
return;
}
if (status && status.ready) {
searchIndexReady = true;
searchIndexStatusBanner.style.display = "none";
searchIndexStatusBanner.textContent = "";
searchIndexStatusBanner.dataset.statusText = "";
if (errorMessageElement) {
errorMessageElement.style.display = "none";
errorMessageElement.textContent = "";
}
return;
}
searchIndexReady = false;
let message = "全文索引正在初始化，请稍候...";
if (status) {
const rawProgress =
typeof status.progress === "number"
? Math.round(status.progress * 100)
: null;
if (status.in_progress) {
message = `全文索引正在构建中，请稍候...`;
} else {
message = `全文索引尚未就绪，请稍候...`;
}
if (status.last_error) {
message += ` 上次错误：${status.last_error}`;
}
}
searchIndexStatusBanner.style.display = "block";
searchIndexStatusBanner.textContent = message;
searchIndexStatusBanner.dataset.statusText = message;
}
function scheduleSearchIndexStatusRefresh(delay) {
if (searchIndexStatusTimer) {
clearTimeout(searchIndexStatusTimer);
}
const wait =
typeof delay === "number" && delay > 0 ? delay : 5000;
searchIndexStatusTimer = setTimeout(() => {
refreshSearchIndexStatus();
}, wait);
}
async function refreshSearchIndexStatus() {
if (!searchIndexStatusBanner) {
searchIndexReady = true;
return;
}
try {
const resp = await fetch(
"/api/v1/search?limit=1&offset=0",
{ headers: { Accept: "application/json" } }
);
if (!resp.ok) {
throw new Error(`HTTP ${resp.status}`);
}
const data = await resp.json();
const status =
data && data.index_status ? data.index_status : null;
updateSearchIndexBanner(status);
if (!searchIndexReady) {
scheduleSearchIndexStatusRefresh(
status && status.in_progress ? 4000 : 8000
);
}
} catch (err) {
console.error("search index status check failed", err);
updateSearchIndexBanner(null, {
error: "无法获取全文索引状态，将稍后重试。",
});
scheduleSearchIndexStatusRefresh(10000);
}
}
refreshSearchIndexStatus();
// 标签切换功能
document.querySelectorAll(".tab").forEach((tab) => {
internal/chatlog/webhook/webhook.go
+4
View File
@@ -166,6 +166,10 @@ func (m *MessageWebhook) Do(event fsnotify.Event) {
return
}
if err := m.db.IndexMessages(messages); err != nil {
log.Warn().Err(err).Msg("incremental fts update failed")
}
m.lastTime = messages[len(messages)-1].Time.Add(time.Second)
for _, message := range messages {
internal/model/avatar.go
+4 -4
View File
@@ -2,8 +2,8 @@ package model
// Avatar represents a user's avatar, either as a remote URL (v3) or inline bytes (v4)
type Avatar struct {
Username string `json:"username"`
URL string `json:"url,omitempty"`
ContentType string `json:"-"`
Data []byte `json:"-"`
Username string `json:"username"`
URL string `json:"url,omitempty"`
ContentType string `json:"-"`
Data []byte `json:"-"`
}
internal/model/intimacy.go
+9 -9
View File
@@ -2,13 +2,13 @@ package model
// IntimacyBase 聚合每个会话/联系人基础统计，用于计算亲密度
type IntimacyBase struct {
UserName string
MsgCount int64
SentCount int64
ReceivedCount int64
MinCreateUnix int64
MaxCreateUnix int64
MessagingDays int64
Last90DaysMsg int64
Past7DaysSentMsg int64
UserName string
MsgCount int64
SentCount int64
ReceivedCount int64
MinCreateUnix int64
MaxCreateUnix int64
MessagingDays int64
Last90DaysMsg int64
Past7DaysSentMsg int64
}
internal/model/stats.go
+9 -9
View File
@@ -2,17 +2,17 @@ package model
// GlobalMessageStats 汇总消息统计
type GlobalMessageStats struct {
Total int64 `json:"total"`
Sent int64 `json:"sent"`
Received int64 `json:"received"`
EarliestUnix int64 `json:"earliest_unix"`
LatestUnix int64 `json:"latest_unix"`
ByType map[string]int64 `json:"by_type"` // 例如：{"文本":123, "图片":456}
Total int64 `json:"total"`
Sent int64 `json:"sent"`
Received int64 `json:"received"`
EarliestUnix int64 `json:"earliest_unix"`
LatestUnix int64 `json:"latest_unix"`
ByType map[string]int64 `json:"by_type"` // 例如：{"文本":123, "图片":456}
}
// MonthlyTrend 月度趋势
type MonthlyTrend struct {
Date string `json:"date"` // YYYY-MM
Sent int64 `json:"sent"`
Received int64 `json:"received"`
Date string `json:"date"` // YYYY-MM
Sent int64 `json:"sent"`
Received int64 `json:"received"`
}
internal/tray/tray.go
+4 -4
View File
@@ -2,12 +2,12 @@ package tray
// Options controls how the tray icon behaves.
type Options struct {
Tooltip string
OnOpen func()
OnQuit func()
Tooltip string
OnOpen func()
OnQuit func()
}
// Controller allows callers to stop the tray icon when shutting down.
type Controller interface {
Stop()
Stop()
}
internal/tray/tray_other.go
+1 -1
View File
@@ -8,5 +8,5 @@ func (noopController) Stop() {}
// Start is a no-op on platforms without a system tray implementation.
func Start(opts Options) (Controller, error) {
return noopController{}, nil
return noopController{}, nil
}
internal/wechatdb/repository/index_incremental.go
+90
View File
@@ -0,0 +1,90 @@
package repository
import (
"context"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/sjzar/chatlog/internal/model"
"github.com/sjzar/chatlog/internal/wechatdb/msgstore"
)
// IndexMessages incrementally indexes the provided messages into the FTS cache.
func (r *Repository) IndexMessages(ctx context.Context, messages []*model.Message) error {
if len(messages) == 0 || r == nil {
return nil
}
if r.index == nil {
return nil
}
r.indexMu.Lock()
status := r.indexStatus
r.indexMu.Unlock()
if status.InProgress || !status.Ready {
return nil
}
batches := make(map[string][]*model.Message)
stores := make(map[string]*msgstore.Store)
for _, msg := range messages {
if msg == nil {
continue
}
store, err := r.ds.LocateMessageStore(msg)
if err != nil {
log.Warn().Err(err).Str("talker", msg.Talker).Msg("locate message store for incremental index failed")
continue
}
if store == nil {
log.Warn().Str("talker", msg.Talker).Msg("skip incremental index: message store not found")
continue
}
batches[store.ID] = append(batches[store.ID], msg)
if _, ok := stores[store.ID]; !ok {
stores[store.ID] = store
}
}
if len(batches) == 0 {
return nil
}
for id, batch := range batches {
store := stores[id]
if len(batch) == 0 || store == nil {
continue
}
if err := r.index.IndexStoreMessages(store, batch); err != nil {
return err
}
}
fp, err := r.ds.GetDatasetFingerprint(ctx)
if err != nil {
log.Debug().Err(err).Msg("get dataset fingerprint for incremental index failed")
return nil
}
if strings.TrimSpace(fp) == "" {
return nil
}
if err := r.index.UpdateFingerprint(fp); err != nil {
return err
}
r.indexMu.Lock()
r.indexFingerprint = fp
r.indexStatus.LastCompletedAt = time.Now()
r.indexMu.Unlock()
return nil
}
internal/wechatdb/wechatdb.go
+7
View File
@@ -82,6 +82,13 @@ func (w *DB) SearchMessages(req *model.SearchRequest) (*model.SearchResponse, er
return w.repo.SearchMessages(ctx, req)
}
func (w *DB) IndexMessages(messages []*model.Message) error {
if w.repo == nil {
return fmt.Errorf("repository not initialized")
}
return w.repo.IndexMessages(context.Background(), messages)
}
type GetContactsResp struct {
Items []*model.Contact `json:"items"`
}
internal/whisper/types.go
+12 -12
View File
@@ -7,18 +7,18 @@ import (
// Options configures a transcription request.
type Options struct {
Language string // "auto" to let the model detect language
LanguageSet bool // true when Language should override defaults
Translate bool // translate non-English speech into English
TranslateSet bool // true when Translate should override defaults
Threads int // number of threads used by the backend (<=0 uses default)
ThreadsSet bool // true when Threads should override defaults
InitialPrompt string // optional priming prompt
InitialPromptSet bool // true when InitialPrompt should override defaults
Temperature float32 // sampling temperature
TemperatureSet bool // true when Temperature should override defaults
TemperatureFloor float32 // optional fallback temperature when decoding stalls
TemperatureFloorSet bool // true when TemperatureFloor should override defaults
Language string // "auto" to let the model detect language
LanguageSet bool // true when Language should override defaults
Translate bool // translate non-English speech into English
TranslateSet bool // true when Translate should override defaults
Threads int // number of threads used by the backend (<=0 uses default)
ThreadsSet bool // true when Threads should override defaults
InitialPrompt string // optional priming prompt
InitialPromptSet bool // true when InitialPrompt should override defaults
Temperature float32 // sampling temperature
TemperatureSet bool // true when Temperature should override defaults
TemperatureFloor float32 // optional fallback temperature when decoding stalls
TemperatureFloorSet bool // true when TemperatureFloor should override defaults
}
// Segment represents a portion of transcribed text with timestamps.
internal/whisper/whispercpp.go
+191 -191
View File
@@ -17,249 +17,249 @@ import (
// WhisperCPPConfig controls the on-device whisper.cpp backend.
type WhisperCPPConfig struct {
ModelPath string
Threads int
DefaultOptions Options
ModelPath string
Threads int
DefaultOptions Options
}
// WhisperCPPTranscriber wraps a whisper.cpp model for local transcription.
type WhisperCPPTranscriber struct {
model whis.Model
defaultOptions Options
defaultThreads int
model whis.Model
defaultOptions Options
defaultThreads int
}
// NewWhisperCPPTranscriber loads a whisper.cpp model for on-device speech recognition.
func NewWhisperCPPTranscriber(cfg WhisperCPPConfig) (*WhisperCPPTranscriber, error) {
modelPath := strings.TrimSpace(cfg.ModelPath)
if modelPath == "" {
return nil, fmt.Errorf("whisper.cpp model path is empty")
}
modelPath := strings.TrimSpace(cfg.ModelPath)
if modelPath == "" {
return nil, fmt.Errorf("whisper.cpp model path is empty")
}
model, err := whis.New(modelPath)
if err != nil {
return nil, fmt.Errorf("load whisper.cpp model: %w", err)
}
model, err := whis.New(modelPath)
if err != nil {
return nil, fmt.Errorf("load whisper.cpp model: %w", err)
}
return &WhisperCPPTranscriber{
model: model,
defaultOptions: cfg.DefaultOptions,
defaultThreads: cfg.Threads,
}, nil
return &WhisperCPPTranscriber{
model: model,
defaultOptions: cfg.DefaultOptions,
defaultThreads: cfg.Threads,
}, nil
}
// Close releases resources held by the whisper.cpp model.
func (t *WhisperCPPTranscriber) Close() {
if t.model != nil {
if err := t.model.Close(); err != nil {
log.Debug().Err(err).Msg("whispercpp: model close failed")
}
t.model = nil
}
if t.model != nil {
if err := t.model.Close(); err != nil {
log.Debug().Err(err).Msg("whispercpp: model close failed")
}
t.model = nil
}
}
// TranscribePCM runs whisper.cpp against raw PCM samples.
func (t *WhisperCPPTranscriber) TranscribePCM(ctx context.Context, samples []float32, sampleRate int, opts Options) (*Result, error) {
if t.model == nil {
return nil, errors.New("whisper.cpp model not initialised")
}
merged := t.mergeOptions(opts)
if t.model == nil {
return nil, errors.New("whisper.cpp model not initialised")
}
merged := t.mergeOptions(opts)
if len(samples) == 0 {
return nil, nil
}
if len(samples) == 0 {
return nil, nil
}
if ctx == nil {
ctx = context.Background()
}
if ctx == nil {
ctx = context.Background()
}
if sampleRate <= 0 {
sampleRate = int(whis.SampleRate)
}
if sampleRate <= 0 {
sampleRate = int(whis.SampleRate)
}
processed := resampleIfNeeded(samples, sampleRate, int(whis.SampleRate))
processed := resampleIfNeeded(samples, sampleRate, int(whis.SampleRate))
ctxInstance, err := t.model.NewContext()
if err != nil {
return nil, fmt.Errorf("create whisper.cpp context: %w", err)
}
ctxInstance, err := t.model.NewContext()
if err != nil {
return nil, fmt.Errorf("create whisper.cpp context: %w", err)
}
threads := t.defaultThreads
if merged.ThreadsSet && merged.Threads > 0 {
threads = merged.Threads
}
if threads > 0 {
ctxInstance.SetThreads(uint(threads))
}
threads := t.defaultThreads
if merged.ThreadsSet && merged.Threads > 0 {
threads = merged.Threads
}
if threads > 0 {
ctxInstance.SetThreads(uint(threads))
}
lang := "auto"
if merged.LanguageSet {
trimmed := strings.TrimSpace(merged.Language)
if trimmed != "" {
lang = trimmed
}
}
if err := ctxInstance.SetLanguage(lang); err != nil {
log.Warn().Err(err).Str("language", lang).Msg("whispercpp: set language failed")
}
lang := "auto"
if merged.LanguageSet {
trimmed := strings.TrimSpace(merged.Language)
if trimmed != "" {
lang = trimmed
}
}
if err := ctxInstance.SetLanguage(lang); err != nil {
log.Warn().Err(err).Str("language", lang).Msg("whispercpp: set language failed")
}
if merged.TranslateSet {
ctxInstance.SetTranslate(merged.Translate)
}
if merged.InitialPromptSet {
ctxInstance.SetInitialPrompt(merged.InitialPrompt)
}
if merged.TemperatureSet {
ctxInstance.SetTemperature(merged.Temperature)
}
if merged.TemperatureFloorSet {
ctxInstance.SetTemperatureFallback(merged.TemperatureFloor)
}
if merged.TranslateSet {
ctxInstance.SetTranslate(merged.Translate)
}
if merged.InitialPromptSet {
ctxInstance.SetInitialPrompt(merged.InitialPrompt)
}
if merged.TemperatureSet {
ctxInstance.SetTemperature(merged.Temperature)
}
if merged.TemperatureFloorSet {
ctxInstance.SetTemperatureFallback(merged.TemperatureFloor)
}
if err := ctxInstance.Process(processed, nil, nil, nil); err != nil {
return nil, fmt.Errorf("whisper.cpp process pcm: %w", err)
}
if err := ctxInstance.Process(processed, nil, nil, nil); err != nil {
return nil, fmt.Errorf("whisper.cpp process pcm: %w", err)
}
if err := ctx.Err(); err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
var (
segments []Segment
builder strings.Builder
lastEnd time.Duration
)
var (
segments []Segment
builder strings.Builder
lastEnd time.Duration
)
for {
seg, err := ctxInstance.NextSegment()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, fmt.Errorf("whisper.cpp next segment: %w", err)
}
for {
seg, err := ctxInstance.NextSegment()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, fmt.Errorf("whisper.cpp next segment: %w", err)
}
text := strings.TrimSpace(seg.Text)
if text == "" {
continue
}
text := strings.TrimSpace(seg.Text)
if text == "" {
continue
}
if builder.Len() > 0 {
builder.WriteByte(' ')
}
builder.WriteString(text)
if builder.Len() > 0 {
builder.WriteByte(' ')
}
builder.WriteString(text)
segments = append(segments, Segment{
ID: seg.Num,
Start: seg.Start,
End: seg.End,
Text: text,
})
if seg.End > lastEnd {
lastEnd = seg.End
}
}
segments = append(segments, Segment{
ID: seg.Num,
Start: seg.Start,
End: seg.End,
Text: text,
})
if seg.End > lastEnd {
lastEnd = seg.End
}
}
detected := strings.TrimSpace(ctxInstance.DetectedLanguage())
if detected == "" {
detected = fallbackLanguage(merged, merged.TranslateSet && merged.Translate)
}
detected := strings.TrimSpace(ctxInstance.DetectedLanguage())
if detected == "" {
detected = fallbackLanguage(merged, merged.TranslateSet && merged.Translate)
}
return &Result{
Text: strings.TrimSpace(builder.String()),
Language: detected,
Duration: lastEnd,
Segments: segments,
}, nil
return &Result{
Text: strings.TrimSpace(builder.String()),
Language: detected,
Duration: lastEnd,
Segments: segments,
}, nil
}
// TranscribeSilk decodes SILK payloads before invoking whisper.cpp.
func (t *WhisperCPPTranscriber) TranscribeSilk(ctx context.Context, silkData []byte, opts Options) (*Result, error) {
if len(silkData) == 0 {
return nil, nil
}
samples16, sampleRate, err := silk.Silk2PCM16(silkData)
if err != nil {
return nil, err
}
if len(silkData) == 0 {
return nil, nil
}
samples16, sampleRate, err := silk.Silk2PCM16(silkData)
if err != nil {
return nil, err
}
floatSamples := make([]float32, len(samples16))
const scale = 1.0 / 32768.0
for i, sample := range samples16 {
floatSamples[i] = float32(float64(sample) * scale)
}
floatSamples := make([]float32, len(samples16))
const scale = 1.0 / 32768.0
for i, sample := range samples16 {
floatSamples[i] = float32(float64(sample) * scale)
}
return t.TranscribePCM(ctx, floatSamples, sampleRate, opts)
return t.TranscribePCM(ctx, floatSamples, sampleRate, opts)
}
func (t *WhisperCPPTranscriber) mergeOptions(overrides Options) Options {
merged := t.defaultOptions
merged := t.defaultOptions
if overrides.LanguageSet {
merged.Language = overrides.Language
merged.LanguageSet = true
}
if overrides.TranslateSet {
merged.Translate = overrides.Translate
merged.TranslateSet = true
}
if overrides.ThreadsSet {
merged.Threads = overrides.Threads
merged.ThreadsSet = true
}
if overrides.InitialPromptSet {
merged.InitialPrompt = overrides.InitialPrompt
merged.InitialPromptSet = true
}
if overrides.TemperatureSet {
merged.Temperature = overrides.Temperature
merged.TemperatureSet = true
}
if overrides.TemperatureFloorSet {
merged.TemperatureFloor = overrides.TemperatureFloor
merged.TemperatureFloorSet = true
}
if overrides.LanguageSet {
merged.Language = overrides.Language
merged.LanguageSet = true
}
if overrides.TranslateSet {
merged.Translate = overrides.Translate
merged.TranslateSet = true
}
if overrides.ThreadsSet {
merged.Threads = overrides.Threads
merged.ThreadsSet = true
}
if overrides.InitialPromptSet {
merged.InitialPrompt = overrides.InitialPrompt
merged.InitialPromptSet = true
}
if overrides.TemperatureSet {
merged.Temperature = overrides.Temperature
merged.TemperatureSet = true
}
if overrides.TemperatureFloorSet {
merged.TemperatureFloor = overrides.TemperatureFloor
merged.TemperatureFloorSet = true
}
return merged
return merged
}
func resampleIfNeeded(samples []float32, fromRate, toRate int) []float32 {
if fromRate <= 0 {
fromRate = toRate
}
if fromRate == toRate || len(samples) == 0 {
dst := make([]float32, len(samples))
copy(dst, samples)
return dst
}
if fromRate <= 0 {
fromRate = toRate
}
if fromRate == toRate || len(samples) == 0 {
dst := make([]float32, len(samples))
copy(dst, samples)
return dst
}
ratio := float64(fromRate) / float64(toRate)
if ratio <= 0 {
dst := make([]float32, len(samples))
copy(dst, samples)
return dst
}
ratio := float64(fromRate) / float64(toRate)
if ratio <= 0 {
dst := make([]float32, len(samples))
copy(dst, samples)
return dst
}
outLen := int(math.Ceil(float64(len(samples)) / ratio))
if outLen <= 0 {
outLen = len(samples)
}
outLen := int(math.Ceil(float64(len(samples)) / ratio))
if outLen <= 0 {
outLen = len(samples)
}
dst := make([]float32, outLen)
for i := range dst {
srcPos := float64(i) * ratio
idx := int(math.Floor(srcPos))
frac := srcPos - float64(idx)
dst := make([]float32, outLen)
for i := range dst {
srcPos := float64(i) * ratio
idx := int(math.Floor(srcPos))
frac := srcPos - float64(idx)
if idx >= len(samples)-1 {
dst[i] = samples[len(samples)-1]
continue
}
if idx >= len(samples)-1 {
dst[i] = samples[len(samples)-1]
continue
}
a := samples[idx]
b := samples[idx+1]
dst[i] = float32(float64(a)*(1-frac) + float64(b)*frac)
}
a := samples[idx]
b := samples[idx+1]
dst[i] = float32(float64(a)*(1-frac) + float64(b)*frac)
}
return dst
}
return dst
}
pkg/util/net.go
-1
View File
@@ -105,4 +105,3 @@ func ComposeLANURL(addr string) string {
}
return "http://" + h + ":" + port
}