434 lines
11 KiB
Go
434 lines
11 KiB
Go
package whisper
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/binary"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
openai "github.com/openai/openai-go/v3"
|
|
"github.com/openai/openai-go/v3/option"
|
|
openaiparam "github.com/openai/openai-go/v3/packages/param"
|
|
"github.com/rs/zerolog/log"
|
|
|
|
"github.com/sjzar/chatlog/pkg/util/silk"
|
|
)
|
|
|
|
// OpenAIConfig describes how to initialise an OpenAI-backed transcriber.
|
|
type OpenAIConfig struct {
|
|
Model string
|
|
TranslateModel string
|
|
APIKey string
|
|
BaseURL string
|
|
Organization string
|
|
ProxyURL string
|
|
RequestTimeout time.Duration
|
|
DefaultOptions Options
|
|
}
|
|
|
|
// OpenAITranscriber uses OpenAI's REST API to perform speech-to-text tasks.
|
|
type OpenAITranscriber struct {
|
|
client *openai.Client
|
|
model openai.AudioModel
|
|
translateModel openai.AudioModel
|
|
defaultOptions Options
|
|
}
|
|
|
|
// NewOpenAITranscriber builds a new instance of the OpenAI transcription backend.
|
|
func NewOpenAITranscriber(cfg OpenAIConfig) (*OpenAITranscriber, error) {
|
|
model := normalizeAudioModel(cfg.Model)
|
|
translateModel := model
|
|
if cfg.TranslateModel != "" {
|
|
translateModel = normalizeAudioModel(cfg.TranslateModel)
|
|
}
|
|
|
|
var opts []option.RequestOption
|
|
if cfg.APIKey != "" {
|
|
opts = append(opts, option.WithAPIKey(cfg.APIKey))
|
|
}
|
|
if cfg.Organization != "" {
|
|
opts = append(opts, option.WithOrganization(cfg.Organization))
|
|
}
|
|
if cfg.BaseURL != "" {
|
|
opts = append(opts, option.WithBaseURL(cfg.BaseURL))
|
|
}
|
|
if cfg.ProxyURL != "" {
|
|
client, err := buildHTTPClient(cfg.ProxyURL, cfg.RequestTimeout)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
opts = append(opts, option.WithHTTPClient(client))
|
|
} else if cfg.RequestTimeout > 0 {
|
|
opts = append(opts, option.WithRequestTimeout(cfg.RequestTimeout))
|
|
}
|
|
|
|
clientVal := openai.NewClient(opts...)
|
|
client := &clientVal
|
|
|
|
return &OpenAITranscriber{
|
|
client: client,
|
|
model: model,
|
|
translateModel: translateModel,
|
|
defaultOptions: cfg.DefaultOptions,
|
|
}, nil
|
|
}
|
|
|
|
func buildHTTPClient(proxyURL string, timeout time.Duration) (*http.Client, error) {
|
|
transport, ok := http.DefaultTransport.(*http.Transport)
|
|
var baseTransport *http.Transport
|
|
if ok {
|
|
baseTransport = transport.Clone()
|
|
} else {
|
|
baseTransport = &http.Transport{Proxy: http.ProxyFromEnvironment}
|
|
}
|
|
|
|
if proxyURL != "" {
|
|
parsed, err := url.Parse(proxyURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid proxy url: %w", err)
|
|
}
|
|
baseTransport.Proxy = http.ProxyURL(parsed)
|
|
}
|
|
|
|
client := &http.Client{
|
|
Transport: baseTransport,
|
|
}
|
|
if timeout > 0 {
|
|
client.Timeout = timeout
|
|
}
|
|
return client, nil
|
|
}
|
|
|
|
// Close releases resources held by the transcriber. No-op for the OpenAI backend.
|
|
func (t *OpenAITranscriber) Close() {}
|
|
|
|
// ModelName returns the audio model identifier currently in use.
|
|
func (t *OpenAITranscriber) ModelName() string {
|
|
return string(t.model)
|
|
}
|
|
|
|
// TranscribePCM converts PCM float32 samples into text via OpenAI's API.
|
|
func (t *OpenAITranscriber) TranscribePCM(ctx context.Context, samples []float32, sampleRate int, opts Options) (*Result, error) {
|
|
merged := t.mergeOptions(opts)
|
|
|
|
if len(samples) == 0 {
|
|
return nil, nil
|
|
}
|
|
if sampleRate <= 0 {
|
|
sampleRate = 24000
|
|
}
|
|
|
|
pcm := float32ToPCM16(samples)
|
|
wav, err := encodePCM16AsWAV(pcm, sampleRate)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return t.transcribeWAV(ctx, wav, sampleRate, len(pcm), merged)
|
|
}
|
|
|
|
// TranscribeSilk converts Silk-encoded payloads into text via OpenAI's API.
|
|
func (t *OpenAITranscriber) TranscribeSilk(ctx context.Context, silkData []byte, opts Options) (*Result, error) {
|
|
merged := t.mergeOptions(opts)
|
|
|
|
if len(silkData) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
samples, sampleRate, err := silk.Silk2PCM16(silkData)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
wav, err := encodePCM16AsWAV(samples, sampleRate)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return t.transcribeWAV(ctx, wav, sampleRate, len(samples), merged)
|
|
}
|
|
|
|
func (t *OpenAITranscriber) transcribeWAV(ctx context.Context, wav []byte, sampleRate, sampleCount int, opts Options) (*Result, error) {
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
|
|
duration := pcmDuration(sampleCount, sampleRate)
|
|
|
|
if opts.TranslateSet && opts.Translate {
|
|
return t.sendTranslation(ctx, wav, opts, duration)
|
|
}
|
|
|
|
return t.sendTranscription(ctx, wav, opts, duration)
|
|
}
|
|
|
|
func (t *OpenAITranscriber) sendTranscription(ctx context.Context, wav []byte, opts Options, fallbackDuration time.Duration) (*Result, error) {
|
|
params := openai.AudioTranscriptionNewParams{
|
|
File: openai.File(bytes.NewReader(wav), "audio.wav", "audio/wav"),
|
|
Model: t.model,
|
|
ResponseFormat: openai.AudioResponseFormatVerboseJSON,
|
|
TimestampGranularities: []string{"segment"},
|
|
}
|
|
|
|
if opts.LanguageSet {
|
|
trimmed := strings.TrimSpace(opts.Language)
|
|
if trimmed != "" && !strings.EqualFold(trimmed, "auto") {
|
|
params.Language = openaiparam.NewOpt(trimmed)
|
|
}
|
|
}
|
|
if opts.InitialPromptSet {
|
|
prompt := strings.TrimSpace(opts.InitialPrompt)
|
|
if prompt != "" {
|
|
params.Prompt = openaiparam.NewOpt(prompt)
|
|
}
|
|
}
|
|
if opts.TemperatureSet {
|
|
params.Temperature = openaiparam.NewOpt(float64(opts.Temperature))
|
|
}
|
|
|
|
transcription, err := t.client.Audio.Transcriptions.New(ctx, params)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return buildResultFromTranscription(transcription, opts, fallbackDuration)
|
|
}
|
|
|
|
func (t *OpenAITranscriber) sendTranslation(ctx context.Context, wav []byte, opts Options, fallbackDuration time.Duration) (*Result, error) {
|
|
params := openai.AudioTranslationNewParams{
|
|
File: openai.File(bytes.NewReader(wav), "audio.wav", "audio/wav"),
|
|
Model: t.translateModel,
|
|
}
|
|
|
|
if opts.InitialPromptSet {
|
|
prompt := strings.TrimSpace(opts.InitialPrompt)
|
|
if prompt != "" {
|
|
params.Prompt = openaiparam.NewOpt(prompt)
|
|
}
|
|
}
|
|
if opts.TemperatureSet {
|
|
params.Temperature = openaiparam.NewOpt(float64(opts.Temperature))
|
|
}
|
|
|
|
translation, err := t.client.Audio.Translations.New(ctx, params)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &Result{
|
|
Text: strings.TrimSpace(translation.Text),
|
|
Language: fallbackLanguage(opts, true),
|
|
Duration: fallbackDuration,
|
|
}, nil
|
|
}
|
|
|
|
func (t *OpenAITranscriber) mergeOptions(overrides Options) Options {
|
|
merged := t.defaultOptions
|
|
|
|
if overrides.LanguageSet {
|
|
merged.Language = overrides.Language
|
|
merged.LanguageSet = true
|
|
}
|
|
if overrides.TranslateSet {
|
|
merged.Translate = overrides.Translate
|
|
merged.TranslateSet = true
|
|
}
|
|
if overrides.ThreadsSet {
|
|
merged.Threads = overrides.Threads
|
|
merged.ThreadsSet = true
|
|
}
|
|
if overrides.InitialPromptSet {
|
|
merged.InitialPrompt = overrides.InitialPrompt
|
|
merged.InitialPromptSet = true
|
|
}
|
|
if overrides.TemperatureSet {
|
|
merged.Temperature = overrides.Temperature
|
|
merged.TemperatureSet = true
|
|
}
|
|
if overrides.TemperatureFloorSet {
|
|
merged.TemperatureFloor = overrides.TemperatureFloor
|
|
merged.TemperatureFloorSet = true
|
|
}
|
|
|
|
return merged
|
|
}
|
|
|
|
func buildResultFromTranscription(tr *openai.Transcription, opts Options, fallbackDuration time.Duration) (*Result, error) {
|
|
if tr == nil {
|
|
return nil, errors.New("openai transcription response is empty")
|
|
}
|
|
|
|
res := &Result{
|
|
Text: strings.TrimSpace(tr.Text),
|
|
Duration: fallbackDuration,
|
|
}
|
|
|
|
raw := tr.RawJSON()
|
|
if raw != "" {
|
|
var payload verboseTranscription
|
|
if err := json.Unmarshal([]byte(raw), &payload); err != nil {
|
|
log.Debug().Err(err).Msg("failed to parse verbose transcription payload")
|
|
} else {
|
|
if payload.Duration > 0 {
|
|
res.Duration = secondsToDuration(payload.Duration)
|
|
}
|
|
if text := strings.TrimSpace(payload.Text); text != "" {
|
|
res.Text = text
|
|
}
|
|
if lang := strings.TrimSpace(payload.Language); lang != "" {
|
|
res.Language = lang
|
|
}
|
|
if len(payload.Segments) > 0 {
|
|
segments := make([]Segment, 0, len(payload.Segments))
|
|
for _, seg := range payload.Segments {
|
|
segments = append(segments, Segment{
|
|
ID: seg.ID,
|
|
Start: secondsToDuration(seg.Start),
|
|
End: secondsToDuration(seg.End),
|
|
Text: strings.TrimSpace(seg.Text),
|
|
})
|
|
}
|
|
res.Segments = segments
|
|
}
|
|
}
|
|
}
|
|
|
|
if res.Duration == 0 {
|
|
res.Duration = fallbackDuration
|
|
}
|
|
if res.Language == "" {
|
|
res.Language = fallbackLanguage(opts, false)
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
func normalizeAudioModel(name string) openai.AudioModel {
|
|
trimmed := strings.TrimSpace(name)
|
|
if trimmed == "" {
|
|
return openai.AudioModelWhisper1
|
|
}
|
|
|
|
lower := strings.ToLower(trimmed)
|
|
if strings.HasSuffix(lower, ".bin") || strings.ContainsAny(trimmed, "\\/") {
|
|
log.Warn().Str("model", trimmed).Msg("ignoring local whisper model path; using OpenAI whisper-1")
|
|
return openai.AudioModelWhisper1
|
|
}
|
|
|
|
return openai.AudioModel(trimmed)
|
|
}
|
|
|
|
func fallbackLanguage(opts Options, translated bool) string {
|
|
if translated {
|
|
return "en"
|
|
}
|
|
if opts.LanguageSet {
|
|
trimmed := strings.TrimSpace(opts.Language)
|
|
if trimmed != "" && !strings.EqualFold(trimmed, "auto") {
|
|
return trimmed
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
type verboseTranscription struct {
|
|
Text string `json:"text"`
|
|
Language string `json:"language"`
|
|
Duration float64 `json:"duration"`
|
|
Segments []verboseTranscriptionSegment `json:"segments"`
|
|
}
|
|
|
|
type verboseTranscriptionSegment struct {
|
|
ID int `json:"id"`
|
|
Start float64 `json:"start"`
|
|
End float64 `json:"end"`
|
|
Text string `json:"text"`
|
|
}
|
|
|
|
func float32ToPCM16(src []float32) []int16 {
|
|
if len(src) == 0 {
|
|
return nil
|
|
}
|
|
dst := make([]int16, len(src))
|
|
for i, sample := range src {
|
|
v := float64(sample)
|
|
if v > 1 {
|
|
v = 1
|
|
} else if v < -1 {
|
|
v = -1
|
|
}
|
|
dst[i] = int16(math.Round(v * 32767))
|
|
}
|
|
return dst
|
|
}
|
|
|
|
func writePCM16AsWAVToWriter(w io.Writer, samples []int16, sampleRate int) error {
|
|
if sampleRate <= 0 {
|
|
sampleRate = 24000
|
|
}
|
|
|
|
dataSize := len(samples) * 2
|
|
riffSize := 36 + dataSize
|
|
byteRate := sampleRate * 2
|
|
blockAlign := 2
|
|
|
|
header := make([]byte, 44)
|
|
copy(header[0:], []byte("RIFF"))
|
|
binary.LittleEndian.PutUint32(header[4:], uint32(riffSize))
|
|
copy(header[8:], []byte("WAVEfmt "))
|
|
binary.LittleEndian.PutUint32(header[16:], 16)
|
|
binary.LittleEndian.PutUint16(header[20:], 1)
|
|
binary.LittleEndian.PutUint16(header[22:], 1)
|
|
binary.LittleEndian.PutUint32(header[24:], uint32(sampleRate))
|
|
binary.LittleEndian.PutUint32(header[28:], uint32(byteRate))
|
|
binary.LittleEndian.PutUint16(header[32:], uint16(blockAlign))
|
|
binary.LittleEndian.PutUint16(header[34:], 16)
|
|
copy(header[36:], []byte("data"))
|
|
binary.LittleEndian.PutUint32(header[40:], uint32(dataSize))
|
|
|
|
if _, err := w.Write(header); err != nil {
|
|
return err
|
|
}
|
|
|
|
payload := make([]byte, len(samples)*2)
|
|
for i, sample := range samples {
|
|
binary.LittleEndian.PutUint16(payload[i*2:], uint16(sample))
|
|
}
|
|
if _, err := w.Write(payload); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func encodePCM16AsWAV(samples []int16, sampleRate int) ([]byte, error) {
|
|
buf := bytes.NewBuffer(nil)
|
|
if err := writePCM16AsWAVToWriter(buf, samples, sampleRate); err != nil {
|
|
return nil, err
|
|
}
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
func pcmDuration(sampleCount int, sampleRate int) time.Duration {
|
|
if sampleRate <= 0 || sampleCount <= 0 {
|
|
return 0
|
|
}
|
|
seconds := float64(sampleCount) / float64(sampleRate)
|
|
return secondsToDuration(seconds)
|
|
}
|
|
|
|
func secondsToDuration(seconds float64) time.Duration {
|
|
if math.IsNaN(seconds) || math.IsInf(seconds, 0) || seconds <= 0 {
|
|
return 0
|
|
}
|
|
return time.Duration(seconds * float64(time.Second))
|
|
}
|