char-fork/llama.cpp

git clone https://git.t4t.associates/char-fork/llama.cpp

Xuan-Son Nguyenmtmd: propagate const to preproc class (#28310)de8656bd9

char/aliceai-t5-moe
brancheschar/aliceai-t5-moeall refs →
6.8 KiB196 linesraw
1#pragma once
2
3#include "ggml.h"
4#include "clip-model.h"
5
6#include <cstdint>
7#include <vector>
8#include <string>
9
10#define MTMD_INTERNAL_HEADER
11
12struct mtmd_audio_mel {
13    int64_t n_len;
14    int64_t n_len_org;
15    int64_t n_mel;
16
17    std::vector<float> data;
18};
19
20struct mtmd_audio_mel_filters {
21    int64_t n_mel;
22    int64_t n_fft;
23
24    std::vector<float> data;
25};
26
27// cache for audio processing, each processor instance owns its own cache
28struct mtmd_audio_cache {
29    std::vector<float> sin_vals;
30    std::vector<float> cos_vals;
31
32    std::vector<float> hann_window;
33
34    mtmd_audio_mel_filters filters;
35
36    void fill_sin_cos_table(uint32_t n);
37
38    void fill_hann_window(uint32_t length, bool periodic);
39
40    // Build mel filterbank matrix [n_mel × n_fft_bins] at runtime.
41    // n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.
42    void fill_mel_filterbank_matrix(int64_t n_mel,
43                                    int64_t n_fft,
44                                    int   sample_rate,               // e.g. 16000
45                                    float fmin             = 0.0f,   // e.g. 0.0
46                                    float fmax             = -1.0f,  // e.g. sr/2; pass -1 for auto
47                                    bool  slaney_area_norm = true,
48                                    float scale            = 1.0f,
49                                    bool  use_htk          = false
50    );
51};
52
53struct mtmd_audio_preprocessor {
54    const clip_hparams & hparams;
55
56    mtmd_audio_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}
57
58    virtual ~mtmd_audio_preprocessor() = default;
59    virtual void initialize() = 0; // NOT thread-safe
60    virtual bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const = 0;
61};
62
63struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
64    mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
65    void initialize() override;
66    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
67
68  private:
69    mtmd_audio_cache cache;
70};
71
72struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
73    mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
74    void initialize() override;
75    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
76
77  private:
78    mtmd_audio_cache cache;
79};
80
81struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
82    mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
83    void initialize() override;
84    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
85
86  private:
87    mtmd_audio_cache cache;
88};
89
90struct mtmd_audio_preprocessor_gemma4a : mtmd_audio_preprocessor {
91    mtmd_audio_preprocessor_gemma4a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
92    void initialize() override;
93    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
94
95  private:
96    mtmd_audio_cache cache;
97};
98
99struct mtmd_audio_preprocessor_gemma4ua : mtmd_audio_preprocessor {
100    mtmd_audio_preprocessor_gemma4ua(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
101    void initialize() override;
102    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
103};
104
105struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor {
106    mtmd_audio_preprocessor_qwen3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
107    void initialize() override;
108    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
109
110  private:
111    mtmd_audio_cache cache;
112};
113
114struct mtmd_audio_preprocessor_dots3note : mtmd_audio_preprocessor {
115    mtmd_audio_preprocessor_dots3note(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
116    void initialize() override;
117    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
118
119  private:
120    mtmd_audio_cache cache;
121};
122
123struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
124    mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
125    void initialize() override;
126    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
127
128  private:
129    mtmd_audio_cache cache;
130};
131
132struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
133    mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
134    void initialize() override;
135    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
136
137  private:
138    mtmd_audio_cache cache;
139};
140
141// mimi convolves the waveform directly, so this only pads it to a whole number of frames
142struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor {
143    mtmd_audio_preprocessor_pockettts(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
144    void initialize() override {}
145    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
146};
147
148struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor {
149    mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { }
150    void initialize() override;
151    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
152
153  private:
154    mtmd_audio_cache cache;
155
156    static void worker_thread(int ith, const float * window_func, int window_size,
157                              const std::vector<float> & samples, int n_samples,
158                              int frame_size, int frame_step, int n_threads,
159                              int n_fft_bins,
160                              const mtmd_audio_cache & cache, mtmd_audio_mel & mel);
161};
162
163//
164// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
165//
166struct mtmd_audio_streaming_istft {
167    mtmd_audio_streaming_istft(int n_fft, int hop_length);
168
169    // reset streaming state
170    void reset();
171
172    // process a single STFT frame (streaming)
173    // frame_spectrum: [n_fft_bins x 2] interleaved real/imag
174    // returns: up to hop_length samples
175    std::vector<float> process_frame(const float * frame_spectrum);
176
177    // flush remaining samples at end of stream
178    std::vector<float> flush();
179
180  private:
181    int n_fft;
182    int hop_length;
183    int n_fft_bins;
184
185    // Own cache for output processing
186    mtmd_audio_cache cache;
187
188    // Streaming state
189    std::vector<float> overlap_buffer;
190    std::vector<float> window_sum_buffer;
191    int                padding_to_remove;
192
193    // Working buffers for IFFT
194    std::vector<float> ifft_in;
195    std::vector<float> ifft_out;
196};