1.对语音数据归一化
如16000hz的数据，会将每个点/32768
2.计算窗函数：（*注意librosa中不进行预处理）
3.进行数据扩展填充，他进行的是镜像填充（"reflect")
如原数据为 12345 -》 填充为4的，左右各填充4 即：5432123454321 即：5432-12345-4321
4.分帧
5.加窗：对每一帧进行加窗，
6.进行fft傅里叶变换
librosa中fft计算，可以使用.net中的System.Numerics
MathNet.Numerics.IntegralTransforms.Fourier.Forward(FFT_frame, FourierOptions.Matlab) 计算，结果相同
7.mel计算（每一帧取20个特征点）
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
|
Imports System.NumericsImports MathNet.NumericsImports MathNet.Numerics.IntegralTransforms Module mfcc_module Public Class Librosa End Class Dim pi As Double = 3.1415926535897931 Public Function spectrum(fft_data(,) As Complex) As Double(,) Dim new_data(fft_data.GetLength(0) - 1, fft_data.GetLength(1) - 1) As Double For n = 0 To fft_data.GetLength(0) - 1 ' Debug.Print("////////////////////////spectrum//////////////////") ' Debug.Print("////////////////////////spectrum//////////////////") For i = 0 To fft_data.GetLength(1) - 1 new_data(n, i) = fft_data(n, i).MagnitudeSquared ' Debug.Write(new_data(n, i) & " ") Next Next Return new_data End Function Public Function FFT(data As Double(,)) As Complex(,) Dim result(data.GetLength(0) - 1, 1024) As Complex '2049 加了一个 数组类型 0 开始 Dim FFT_frame As Complex() = New Complex(data.GetLength(1) - 1) {} For n = 0 To data.GetLength(0) - 1 For i As Integer = 0 To data.GetLength(1) - 1 FFT_frame(i) = data(n, i) Next MathNet.Numerics.IntegralTransforms.Fourier.Forward(FFT_frame, FourierOptions.Matlab) For k = 0 To 1024 result(n, k) = FFT_frame(k) Next 'Debug.Print("fft **************") 'For Each mem In FFT_frame ' Debug.Print(mem.ToString & " ") 'Next Next n Return result End Function Public Function _mfcc(dct_ As Double(,）, power_to_db_ As Double(,)) As Double(,) 'dct 20,128 'power_to_db 5,128 'result = 20,5 Dim result(dct_.GetLength(0) - 1, power_to_db_.GetLength(1) - 1) As Double Dim r1, r2 As Double For n = 0 To dct_.GetLength(0) - 1 '20 For i = 0 To power_to_db_.GetLength(1) - 1 '5 r2 = 0 For k = 0 To dct_.GetLength(1) - 1 '128 r1 = dct_(n, k) * power_to_db_(k, i) r2 = r2 + r1 Next result(n, i) = r2 Next Next Return result End Function Public Function Dct(n_filters As Integer, n_input As Integer) As Double(,) Dim t1 As Double = 2 * n_input Dim samples(n_input - 1) As Double Dim basis(n_filters - 1, n_input - 1） As Double Dim n As Integer = 1 For i = 0 To n_input - 1 samples(i) = n * pi / (2 * n_input) n = n + 2 Next i For i = 0 To n_input - 1 basis(0, i) = 1 / Math.Sqrt(n_input) Next For n = 1 To n_filters - 1 For i = 0 To n_input - 1 basis(n, i) = Math.Cos(n * samples(i)) * Math.Sqrt(2 / n_input) Next Next Return basis End Function '1e-10 = 0.0000000001 Public Function power_to_db(S As Double(,), Optional ref As Double = 1, Optional admin As Double = 0.0000000001, Optional top_db As Double = 80) As Double(,) Dim result(S.GetLength(0) - 1, S.GetLength(1) - 1） As Double Dim log_spec As Double For n = 0 To S.GetLength(0) - 1 For i = 0 To S.GetLength(1) - 1 log_spec = 10 * Math.Log10(Math.Max(admin, S(n, i))) result(n, i) = log_spec - 10 * Math.Log10(Math.Max(admin, ref)) Next Next 'If top_db <> 0 Then ' For n = 0 To S.GetLength(0) - 1 ' For i = 0 To S.GetLength(1) - 1 ' 'result(n, i) = Math.Max(result(n, i), result(n, i) - top_db) ' Next ' Next 'End If Return result End Function Public Function melspectrogram(mel_basis(,) As Double, s(,) As Double) As Double(,) 'mel_basis 128,1025 's 5 ,1025 -> 1025,5 ' result 128,5 Dim result(mel_basis.GetLength(0) - 3, s.GetLength(0) - 1) As Double Dim r1, r2 As Double For n = 0 To mel_basis.GetLength(0) - 3 For i = 0 To s.GetLength(0) - 1 For k = 0 To mel_basis.GetLength(1) - 1 r1 = mel_basis(n, k) * s(i, k) r2 = r2 + r1 Next result(n, i) = r2 r2 = 0 Next Next Return result End Function Public Function normal(mel_f As Double(), weights(,) As Double) As Double(,) Dim enorm(mel_f.Length - 2) As Double ' Debug.Print("*************normal//////////////") ' Debug.Print("*************normal//////////////") For i = 0 To mel_f.Length - 3 enorm(i) = 2 / (mel_f(2 + i) - mel_f(i)) Next For i = 0 To weights.GetLength(1) - 1 For n = 0 To weights.GetLength(0) - 2 weights(n, i) = weights(n, i) * enorm(n) Next Next Return weights End Function Public Function weight(a As Double(,), fdiff As Double()) As Double(,) Dim lower, upper As Double Dim data(a.GetLength(0) - 1, a.GetLength(1) - 1) As Double For n = 0 To a.GetLength(0) - 3 For i = 0 To a.GetLength(1) - 1 lower = -(a(n, i) / fdiff(n)) upper = a(n + 2, i) / fdiff(n + 1) data(n, i) = Math.Max(0, Math.Min(lower, upper)) Next Next Return data End Function Public Function ramps(A As Double(), B As Double()) As Double(,) Dim data(A.Length - 1, B.Length - 1) As Double ' Debug.Print("ramps*********************") For n = 0 To A.Length - 1 'Debug.Print("******") 'Debug.Print("------") For i = 0 To B.Length - 1 data(n, i) = A(n) - B(i) 'Debug.Write(data(n, i) & " ") Next Next Return data End Function Public Function diff(arr As Double()) As Double() Dim data(arr.Length - 2) As Double For i = 1 To arr.Length - 1 data(i - 1) = arr(i) - arr(i - 1) 'Debug.Print(data(i - 1)) Next Return data End Function '分帧 算法2 Public Function Frame2(y As Double(), Optional n_ftt As Integer = 2048, Optional hop As Integer = 512) As Double(,) Dim tim As Integer = Math.Floor((y.Length - n_ftt) / hop) + 1 Dim new_buff(tim - 1, n_ftt - 1) As Double Dim copypos As Integer = 0 For i = 0 To tim - 1 For k = 0 To n_ftt - 1 new_buff(i, k) = y(copypos + k) Next copypos = copypos + hop Next 'For k = 0 To tim - 1 ' Debug.Print("//////////////////////////////////////") ' Debug.Print("///////////////fram2///////////////////////" & k) ' For i = 0 To n_ftt - 1 ' Debug.Print(new_buff(k, i) & " ") ' Next 'Next k Return new_buff End Function ' Public Function Frame(y As Double(), Optional n_ftt As Integer = 2048, Optional hop As Integer = 512) As Double() Dim tim As Integer = Math.Floor((y.Length - n_ftt) / hop) + 1 Dim new_buff(tim * n_ftt) As Double Dim pos As Integer = 0 Dim copypos As Integer = 0 For i = 0 To tim - 1 Array.Copy(y, copypos, new_buff, pos, n_ftt) 'Buffer.BlockCopy(y, 0, new_buff, pos, n_ftt) copypos = copypos + hop pos = pos + n_ftt Next For k = 0 To tim - 1 'Debug.Print("//////////////////////////////////////") 'Debug.Print("//////////////////////////////////////") For i = 0 To n_ftt - 1 Debug.Write(new_buff(k * n_ftt + i) & " ") Next Next k Return new_buff End Function Public Function MelFilter() As Double() Dim filter_points（128 + 1） As Integer '40个滤波器，需要41点 Const sampleRate As Integer = 16000 '采样频率 16000 Const filterNum As Integer = 128 '滤波器数量 取40个 Const frameSize As Integer = 512 '帧长512 Dim freMax As Double = sampleRate / 2 '实际最大频率 Dim freMin As Double = 0 '实际最小频率 Dim melFremax As Double = hz_to_mel(freMax) '将实际频率转换成梅尔频率 Dim melFremin As Double = 1125 * Math.Log(1 + freMin / 700) Dim k As Double = (melFremax - melFremin) / (filterNum + 1) Dim m As Double() = New Double(filterNum + 1) {} Dim h As Double() = New Double(filterNum + 1) {} For i As Integer = 0 To filterNum + 1 m(i) = melFremin + k * i 'h(i) = 700 * (Math.Exp(m(i) / 1125) - 1) '将梅尔频率转换成实际频率 filter_points(i) = mel_to_hz(m(i)) 'Debug.Print(m(i)) Next Dim hzs As Double() = mel_to_hz2(m) 'For i = 0 To filterNum + 1 ' ' Debug.Print(hzs(i)) 'Next Return hzs End Function Public Function hz_to_mel(frequencies As Double, Optional htk As Boolean = False) As Double Dim mels As Double If htk Then mels = 1125 * Math.Log(1 + frequencies / 700) Else Dim f_min As Double = 0.0 Dim f_sp As Double = 200.0 / 3 Dim min_log_hz As Double = 1000.0 ' beginning of log region (Hz) Dim min_log_mel As Double = (min_log_hz - f_min) / f_sp ' same (Mels) Dim logstep As Double = Math.Log(6.4) / 27.0 ' step size for log region mels = min_log_mel + Math.Log(frequencies / min_log_hz) / logstep End If Return mels End Function Public Function mel_to_hz2(mel() As Double, Optional htk As Boolean = False) As Double() Dim hz(mel.Length - 1) As Double Dim f_min As Double = 0.0 Dim f_sp As Double = 200.0 / 3 Dim freqs(mel.Length - 1) As Double For i = 0 To mel.Length - 1 freqs(i) = f_min + f_sp * mel(i) Next i Dim min_log_hz As Double = 1000.0 ' beginning of log region (Hz) Dim min_log_mel As Double = (min_log_hz - f_min) / f_sp ' same (Mels) Dim logstep As Double = Math.Log(6.4) / 27.0 For i = 0 To mel.Length - 1 If (mel(i) > min_log_mel) Then freqs(i) = min_log_hz * Math.Exp(logstep * (mel(i) - min_log_mel)) End If Next 'hz = min_log_hz * Math.Exp(logstep * (mel - min_log_mel)) Return freqs End Function Public Function mel_to_hz(mel As Double, Optional htk As Boolean = False) As Double Dim hz As Double If htk Then hz = 700 * (Math.Exp(mel) / 1125) - 1 Else Dim f_min As Double = 0.0 Dim f_sp As Double = 200.0 / 3 Dim freqs = f_min + f_sp * mel Dim min_log_hz As Double = 1000.0 ' beginning of log region (Hz) Dim min_log_mel As Double = (min_log_hz - f_min) / f_sp ' same (Mels) Dim logstep As Double = Math.Log(6.4) / 27.0 hz = min_log_hz * Math.Exp(logstep * (mel - min_log_mel)) 'hz = min_log_hz * Math.Exp(logstep * (mel - min_log_mel)) End If Return hz End Function Public Function fft_frequencies(sr As Integer, n_fft As Integer) As Double() Dim fft_data(n_fft / 2) As Double For i = 0 To n_fft / 2 fft_data(i) = i * sr / n_fft Next Return fft_data End Function '左右填充，优化 Public Function PadReflect2(data() As Double, num As Integer) 'pad 10 ,10 Dim tim(data.Length - 3) As Double For i = 0 To data.Length - 3 tim(i) = data(data.Length - 2 - i) Next Dim dump() As Double = data.Concat(tim).ToArray() 'For Each i In dump ' Debug.Write(i) End Function Public Function PadReflect(data() As Double, num As Integer) 'pad 10 ,10 Dim tim(data.Length - 3) As Double For i = 0 To data.Length - 3 tim(i) = data(data.Length - 2 - i) Next Dim dump() As Double = data.Concat(tim).ToArray() 'For Each i In dump ' Debug.Write(i) 'Next 'left_edge ' Debug.Print("***************************") Dim left_edge(num - 1) As Double _CopyDup(left_edge, dump, True) 'For i = 0 To num - 1 ' Debug.Write(left_edge(i)) 'Next 'right_edge 'Debug.Print("***************************") Dim right_edge(num + data.Length) As Double _CopyDup(right_edge, dump, False) 'For i = 0 To num - 1 ' Debug.Write(right_edge(i)) 'Next 'Debug.Print("***************************") Dim result As Double() = left_edge.Concat(right_edge).ToArray() Return result End Function 'copy tim to data dumply Public Function _CopyDup(data() As Double, tim() As Double, Optional left As Boolean = True) Dim last As Integer = data.Length Mod tim.Length Dim times As Integer = Math.Floor(data.Length / tim.Length) Dim pos As Integer If left Then Array.Copy(tim, tim.Length - last, data, 0, last) pos = last For i = 0 To times - 1 Array.Copy(tim, 0, data, pos, tim.Length) pos = pos + tim.Length Next Else 'RIGHT pos = 0 For i = 0 To times - 1 Array.Copy(tim, 0, data, pos, tim.Length) pos = pos + tim.Length Next Array.Copy(tim, 0, data, pos, last) End If End Function Public Function General_cosine(M As Integer, alpha As Double(), sym As Boolean) As Double() If Not sym Then M = M + 1 End If Dim tim As Double = (2 * pi) / (M - 1) Dim x(M) As Double Dim w(M) As Double 'Debug.Print("ine") For i = 0 To M - 1 x(i) = -pi + tim * i 'Debug.Write(x(i) & " ") Next 'Debug.Print("******") For i = 0 To alpha.GetLength(0) - 1 For k = 0 To M - 1 w(k) = w(k) + alpha(i) * Math.Cos(i * x(k)) 'Debug.Write(w(k) & " ") Next Next Return w End Function ''' <summary> ''' 汉明窗 ''' </summary> ''' <param name="M"> 窗长</param> ''' <returns></returns> Public Function General_hamming(M As Integer) As Double() Dim db As Double() = {0.5, 1 - 0.5} Return General_cosine(M, db, False) '进行加1 ，若sys为false End Function Public Function Get_window(M As Integer) As Double() Return General_hamming(M) End Function End Module |
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
|
Imports System.IOImports System.NumericsImports TensorFlow 'Install-Package TensorFlowSharp Public Class KeyWordDetect Dim graph As TFGraph Dim session As TFSession '加载模型 Public Sub New() Dim model As Byte() = File.ReadAllBytes("f:\graph1.pb") '导入GraphDef graph = New TFGraph() graph.Import(model, "") session = New TFSession(graph) ' Threading.ThreadPool.SetMaxThreads(5, 5) End Sub Protected Overrides Sub finalize() session.CloseSession() End Sub '将声音数据变为mfcc byte数据 Public Function DataBToMFCC(dataB() As Byte) As Double(,) Dim buff16(dataB.Length / 2 - 1) As Int16 Buffer.BlockCopy(dataB, 0, buff16, 0, dataB.Length - 1) Dim result(,) As Double = MFCC(buff16) Return result End Function '将声音数据变为mfcc Public Function DataToMFCC(dataI() As Int16) As Double(,) Dim result(,) As Double = MFCC(dataI) Return result End Function '将mfcc变为输入数据格式 Public Function MFCCToVect(mfcc As Double(,)) As Double(,,) Dim data(0, 1, 129) As Double Dim n As Integer = 0, m As Integer = 0 For i = 0 To mfcc.GetLength(0) - 1 For k = 0 To mfcc.GetLength(1) - 1 data(0, m, n) = mfcc(i, k) n = n + 1 Next If n = 130 Then m = 1 n = 0 End If Next Return data End Function Dim output Dim runner As TFSession.Runner Dim result Dim rshape '关键字检测 Public Function Detected(Data(,,) As Double) As Double ' Dim tensor As TFTensor = New TFTensor(Data) runner = session.GetRunner() runner.AddInput(graph("input")(0), Data).Fetch(graph("out")(0)) output = runner.Run() result = output(0) rshape = result.Shape Dim rt As Double rt = result.GetValue(True)(0)(0) 'For k = 0 To rshape.GetValue(0) - 1 ' rt = result.GetValue(True)(k)(0) ' 'Debug.Print(rt) ' If (rt > 0.8) Then ' Debug.Print("-----------recogxili") ' ' MsgBox("recgo") ' End If 'Next Return RT End Function 'Public Function RunB(dataB() As Byte) ' Dim mfccd As Double(,) = DataBToMFCC(dataB) ' Dim inputx As Double(,,) = MFCCToVect(mfccd) ' Detected(inputx) 'End Function 'Public Function ThreadPoolRun(dataI() As Int16) ' Threading.ThreadPool.QueueUserWorkItem(Run(dataI), dataI) ' ' Dim thrd1 As New Threading.Thread(New Threading.ParameterizedThreadStart(AddressOf Run)) ' ' thrd1.Start(dataI) 'End Function 'Delegate Function DelgRun(dataI() As Int16) 'Public Function ThreadRun(dataI() As Int16) ' ' Dim drun As New DelgRun(AddressOf Run) ' Dim thrd1 As New Threading.Thread(New Threading.ParameterizedThreadStart(AddressOf Run)) ' thrd1.Start(dataI) 'End Function Public Function Run(dataI() As Int16) As Double ' Debug.Print("thread *****1") Dim mfccd As Double(,) = DataToMFCC(dataI) Dim inputx As Double(,,) = MFCCToVect(mfccd) Return Detected(inputx) End Function Public Function MFCC(buff16() As Int16) As Double(,) Dim datalen As Integer = buff16.Length * 2 Dim double_buff(datalen / 2 - 1) As Double Dim len As Integer = datalen / 2 Array.Copy(buff16, double_buff, len) '****************** For i = 0 To double_buff.Length - 1 double_buff(i) = double_buff(i) / 32768 ' Debug.Print(double_buff(i)) Next '汉明窗create Dim hann_window As Double() = Get_window(2048) 'Debug.Print("--------------------------") 'Debug.Print("hann_window**************") For Each i In hann_window 'Debug.Print(i & " ") Next 'Debug.Print("--------------------------") 'Debug.Print("*************pad reflect**************") Dim y As Double() = PadReflect(double_buff, 1024) ' Dim y As Double() = double_buff 'For Each i In y ' 'Debug.Print(i & " ") 'Next 'Debug.Print("--------------------------") 'Debug.Print("***************frame************") Dim frams As Double(,) = Frame2(y) Dim tim As Integer = frams.GetLength(0) 'Debug.Print("--------------------------") 'Debug.Print("**********hann * data**************") Dim hannData(tim - 1, 2047) As Double For n = 0 To tim - 1 For i = 0 To 2048 - 1 hannData(n, i) = frams(n, i) * hann_window(i) ' Debug.Print(hannData(i) & " ") Next Next n '\\\\\\\\\\\\\\\\melspecture Dim specturm1(,) As Complex = FFT(hannData) 'For i = 0 To specturm1.GetLength(0) - 1 ' Debug.Print("--------------------------------------") ' Debug.Print("--------------------------------------") ' For k = 0 To specturm1.GetLength(1) - 1 ' Debug.Print(specturm1（i, k).Real & " " & specturm1(i, k).Imaginary) ' Next 'Next Dim s As Double(,) = spectrum(specturm1) Dim fftfreqs() As Double = fft_frequencies(16000, 2048) 'Debug.Print("***************fftfreqs*****************") 'Debug.Print("***************fftfreqs*****************") 'Debug.Print("fftfreqs.shape", fftfreqs.Length) 'For i = 0 To fftfreqs.Length - 1 ' 'Debug.Write(fftfreqs(i) & " ") 'Next ''''''''''''''''mel * specturm1 'Debug.Print("**************") 'Debug.Print("****滤波器创建**********") Dim mel_f As Double() = MelFilter() 'Debug.Print("--------------------------") 'Debug.Print("hann_window**************") 'Debug.Print("diff") Dim fdiff As Double() = diff(mel_f) Dim ramps_ As Double(,) = ramps(mel_f, fftfreqs) Dim weights(,) As Double = weight(ramps_, fdiff) normal(mel_f, weights) 'S*WEIGHT = melspectrogram 'weight 128,1025 's 5 ,1025 Dim melspectrogram_(,) As Double = melspectrogram(weights, s) Dim power_to_db_ As Double（,) = power_to_db(melspectrogram_) Dim dct_ As Double(,) = Dct(20, 128) Return _mfcc(dct_, power_to_db_) End Function End Class |
以上这篇对python中Librosa的mfcc步骤详解就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持服务器之家。
原文链接：https://blog.csdn.net/qq_39239990/article/details/80876250
