鸿蒙AI能力之语音识别

文章旨在帮助大家开发录音及语音识别时少踩一点坑。

效果

左侧为简易UI布局及识别成果，右侧为网易云播放的测试音频。

开发步骤

IDE安装、项目创建等在此略过。App采用SDK版本为API 6，使用JS UI。

1.权限申请

AI语音识别不需要任何权限，但此处使用到麦克风录制音频，就需要申请麦克风权限。

在config.json配置文件中添加权限：

"reqPermissions": [
{
"name": "ohos.permission.MICROPHONE"
}
]

在MainAbility中显示申明麦克风权限：

@Override
public void onStart(Intent intent) {
super.onStart(intent);
requestPermission();
}
//获取权限
private void requestPermission() {
String[] permission = {
"ohos.permission.MICROPHONE",
};
List applyPermissions = new ArrayList<>();
for (String element : permission) {
if (verifySelfPermission(element) != 0) {
if (canRequestPermission(element)) {
applyPermissions.add(element);
}
}
}
requestPermissionsFromUser(applyPermissions.toArray(new String[0]), 0);
}

2.创建音频录制的工具类

首先创建音频录制的工具类AudioCaptureUtils。

而音频录制需要用到AudioCapturer类，而在创建AudioCapture类时又会用到AudioStreamInfo类及AudioCapturerInfo类，所以我们分别申明以上3个类的变量。

private AudioStreamInfo audioStreamInfo;
private AudioCapturer audioCapturer;
private AudioCapturerInfo audioCapturerInfo;

在语音识别时对音频的录制是由限制的，限制如下：

所以我们在录制音频时需要注意：

1.采样率16000HZ

2.声道为单声道

3.仅支持普通话

作为工具类，为了使AudioCaptureUtils能多处使用，我们在创建构造函数时，提供声道与频率的参数重载,并在构造函数中初始化AudioStreamInfo类及AudioCapturerInfo类。

//channelMask 声道
//SampleRate 频率
public AudioCaptureUtils(AudioStreamInfo.ChannelMask channelMask, int SampleRate) {
this.audioStreamInfo = new AudioStreamInfo.Builder()
.encodingFormat(AudioStreamInfo.EncodingFormat.ENCODING_PCM_16BIT)
.channelMask(channelMask)
.sampleRate(SampleRate)
.build();
this.audioCapturerInfo = new AudioCapturerInfo.Builder().audioStreamInfo(audioStreamInfo).build();
}

在init函数中进行audioCapturer的初始化，在初始化时对音效进行设置，默认为降噪模式。

//packageName 包名
public void init(String packageName) {
this.init(SoundEffect.SOUND_EFFECT_TYPE_NS,packageName );
}
//soundEffect 音效uuid
//packageName 包名
public void init(UUID soundEffect, String packageName) {
if (audioCapturer == null || audioCapturer.getState() == AudioCapturer.State.STATE_UNINITIALIZED)
audioCapturer = new AudioCapturer(this.audioCapturerInfo);
audioCapturer.addSoundEffect(soundEffect, packageName);
}

初始化后提供start、stop和destory方法，分别开启音频录制、停止音频录制和销毁,此处都是调用AudioCapturer类中对应函数。

public void stop(){
this.audioCapturer.stop();
}
public void destory(){
this.audioCapturer.stop();
this.audioCapturer.release();
}
public Boolean start() {
if (audioCapturer == null)
return false;
return audioCapturer.start();
}

提供一个读取音频流的方法及获取AudioCapturer实例的方法。

//buffers 需要写入的数据流
//offset 数据流的偏移量
//byteslength 数据流的长度
public int read(byte[] buffers, int offset, int bytesLength){
return audioCapturer.read(buffers,offset,bytesLength);
}
//获取AudioCapturer的实例audioCapturer
public AudioCapturer get(){
return this.audioCapturer;
}

3.创建语音识别的工具类

在上面我们已经创建好一个音频录制的工具类，接下来在创建一个语音识别的工具类 AsrUtils。

我们再回顾一下语音识别的约束与限制：

在此补充一个隐藏限制，PCM流的长度只允许640与1280两种长度，也就是我们音频读取流时只能使用640与1280两种长度。

接下来我们定义一些基本常量：

//采样率限定16000HZ
private static final int VIDEO_SAMPLE_RATE = 16000;
//VAD结束时间默认2000ms
private static final int VAD_END_WAIT_MS = 2000;
//VAD起始时间默认4800ms
//这两参数与识别准确率有关，相关信息可百度查看，在此使用系统默认
private static final int VAD_FRONT_WAIT_MS = 4800;
//输入时常 20000ms
private static final int TIMEOUT_DURATION = 20000;
//PCM流长度仅限640或1280
private static final int BYTES_LENGTH = 1280;
//线程池相关参数
private static final int CAPACITY = 6;
private static final int ALIVE_TIME = 3;
private static final int POOL_SIZE = 3;

因为要在后台持续录制音频，所以需要开辟一个新的线程。此处用到java的ThreadPoolExecutor类进行线程操作。

定义一个线程池实例以及其它相关属性如下：

//录音线程
private ThreadPoolExecutor poolExecutor;
/* 自定义状态信息
** 错误：-1
** 初始：0
** init:1
** 开始输入:2
** 结束输入:3
** 识别结束:5
** 中途出识别结果:9
** 最终识别结果：10
*/
public int state = 0;
//识别结果
public String result;
//是否开启语音识别
//当开启时才写入PCM流
boolean isStarted = false;
//ASR客户端
private AsrClient asrClient;
//ASR监听对象
private AsrListener listener;
AsrIntent asrIntent;
//音频录制工具类
private AudioCaptureUtils audioCaptureUtils;

在构造函数中初始化相关属性：

public AsrUtils(Context context) {
//实例化一个单声道，采集频率16000HZ的音频录制工具类实例
this.audioCaptureUtils = new AudioCaptureUtils(AudioStreamInfo.ChannelMask.CHANNEL_IN_MONO, VIDEO_SAMPLE_RATE);
//初始化降噪音效
this.audioCaptureUtils.init("com.panda_coder.liedetector");
//结果值设为空
this.result = "";
//给录音控件初始化一个新的线程池
poolExecutor = new ThreadPoolExecutor(
POOL_SIZE,
POOL_SIZE,
ALIVE_TIME,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(CAPACITY),
new ThreadPoolExecutor.DiscardOldestPolicy());
if (asrIntent == null) {
asrIntent = new AsrIntent();
//设置音频来源为PCM流
//此处也可设置为文件
asrIntent.setAudioSourceType(AsrIntent.AsrAudioSrcType.ASR_SRC_TYPE_PCM);
asrIntent.setVadEndWaitMs(VAD_END_WAIT_MS);
asrIntent.setVadFrontWaitMs(VAD_FRONT_WAIT_MS);
asrIntent.setTimeoutThresholdMs(TIMEOUT_DURATION);
}
if (asrClient == null) {
//实例化AsrClient
asrClient = AsrClient.createAsrClient(context).orElse(null);
}
if (listener == null) {
//实例化MyAsrListener
listener = new MyAsrListener();
//初始化AsrClient
this.asrClient.init(asrIntent, listener);
}
}
//够建一个实现AsrListener接口的类MyAsrListener
class MyAsrListener implements AsrListener {
@Override
public void onInit(PacMap pacMap) {
HiLog.info(TAG, "====== init");
state = 1;
}
@Override
public void onBeginningOfSpeech() {
state = 2;
}
@Override
public void onRmsChanged(float v) {
}
@Override
public void onBufferReceived(byte[] bytes) {
}
@Override
public void onEndOfSpeech() {
state = 3;
}
@Override
public void onError(int i) {
state = -1;
if (i == AsrError.ERROR_SPEECH_TIMEOUT) {
//当超时时重新监听
asrClient.startListening(asrIntent);
} else {
HiLog.info(TAG, "======error code:" + i);
asrClient.stopListening();
}
}
//注意与onIntermediateResults获取结果值的区别
//pacMap.getString(AsrResultKey.RESULTS_RECOGNITION);
@Override
public void onResults(PacMap pacMap) {
state = 10;
//获取最终结果
//{"result":[{"confidence":0,"ori_word":"你好 ","pinyin":"NI3 HAO3 ","word":"你好。"}]}
String results = pacMap.getString(AsrResultKey.RESULTS_RECOGNITION);
ZSONObject zsonObject = ZSONObject.stringToZSON(results);
ZSONObject infoObject;
if (zsonObject.getZSONArray("result").getZSONObject(0) instanceof ZSONObject) {
infoObject = zsonObject.getZSONArray("result").getZSONObject(0);
String resultWord = infoObject.getString("ori_word").replace(" ", "");
result += resultWord;
}
}
//中途识别结果
//pacMap.getString(AsrResultKey.RESULTS_INTERMEDIATE)
@Override
public void onIntermediateResults(PacMap pacMap) {
state = 9;
// String result = pacMap.getString(AsrResultKey.RESULTS_INTERMEDIATE);
// if (result == null)
// return;
// ZSONObject zsonObject = ZSONObject.stringToZSON(result);
// ZSONObject infoObject;
// if (zsonObject.getZSONArray("result").getZSONObject(0) instanceof ZSONObject) {
// infoObject = zsonObject.getZSONArray("result").getZSONObject(0);
// String resultWord = infoObject.getString("ori_word").replace(" ", "");
// HiLog.info(TAG, "=========== 9 " + resultWord);
// }
}
@Override
public void onEnd() {
state = 5;
//当还在录音时，重新监听
if (isStarted)
asrClient.startListening(asrIntent);
}
@Override
public void onEvent(int i, PacMap pacMap) {
}
@Override
public void onAudioStart() {
state = 2;
}
@Override
public void onAudioEnd() {
state = 3;
}
}

开启识别与停止识别的函数：

public void start() {
if (!this.isStarted) {
this.isStarted = true;
asrClient.startListening(asrIntent);
poolExecutor.submit(new AudioCaptureRunnable());
}
}
public void stop() {
this.isStarted = false;
asrClient.stopListening();
audioCaptureUtils.stop();
}
//音频录制的线程
private class AudioCaptureRunnable implements Runnable {
@Override
public void run() {
byte[] buffers = new byte[BYTES_LENGTH];
//开启录音
audioCaptureUtils.start();
while (isStarted) {
//读取录音的PCM流
int ret = audioCaptureUtils.read(buffers, 0, BYTES_LENGTH);
if (ret <= 0) {
HiLog.error(TAG, "======Error read data");
} else {
//将录音的PCM流写入到语音识别服务中
//若buffer的长度不为1280或640时，则需要手动处理成1280或640
asrClient.writePcm(buffers, BYTES_LENGTH);
}
}
}
}

识别结果是通过listener的回调获取的结果，所以我们在处理时是将结果赋值给result，通过getresult或getResultAndClear函数获取结果。

public String getResult() {
return result;
}
public String getResultAndClear() {
if (this.result == "")
return "";
String results = getResult();
this.result = "";
return results;
}

4.创建一个简易的JS UI，并通过JS调ServerAbility的能力调用Java

hml代码：

"container">
"title">
语音识别内容： {{ text }}

样式代码：

.container {
flex-direction: column;
justify-content: flex-start;
align-items: center;
width: 100%;
height: 100%;
padding: 10%;
}
.title {
font-size: 20px;
color: #000000;
opacity: 0.9;
text-align: left;
width: 100%;
margin: 3% 0;
}
.btn{
padding: 10px 20px;
margin:3px;
border-radius: 6px;
}

js逻辑控制代码：

//js调Java ServiceAbility的工具类
import { jsCallJavaAbility } from '../../common/JsCallJavaAbilityUtils.js';
export default {
data: {
text: ""
},
//开启事件
start() {
jsCallJavaAbility.callAbility("ControllerAbility",100,{}).then(result=>{
console.log(result)
})
},
//关闭事件
stop() {
jsCallJavaAbility.callAbility("ControllerAbility",101,{}).then(result=>{
console.log(result)
})
jsCallJavaAbility.unSubAbility("ControllerAbility",201).then(result=>{
if (result.code == 200) {
console.log("取消订阅成功");
}
})
},
//订阅Java端结果事件
sub() {
jsCallJavaAbility.subAbility("ControllerAbility", 200, (data) => {
let text = data.data.text
text && (this.text += text)
}).then(result => {
if (result.code == 200) {
console.log("订阅成功");
}
})
}
}

ServerAbility：

public class ControllerAbility extends Ability {
AnswerRemote remote = new AnswerRemote();
AsrUtils asrUtils;
//订阅事件的委托
private static HashMap<Integer, IRemoteObject> remoteObjectHandlers = new HashMap<Integer, IRemoteObject>();
@Override
public void onStart(Intent intent) {
HiLog.error(LABEL_LOG, "ControllerAbility::onStart");
super.onStart(intent);
//初始化语音识别工具类
asrUtils = new AsrUtils(this);
}
@Override
public void onCommand(Intent intent, boolean restart, int startId) {
}
@Override
public IRemoteObject onConnect(Intent intent) {
super.onConnect(intent);
return remote.asObject();
}
class AnswerRemote extends RemoteObject implements IRemoteBroker {
AnswerRemote() {
super("");
}
@Override
public boolean onRemoteRequest(int code, MessageParcel data, MessageParcel reply, MessageOption option) {
Map zsonResult = new HashMap();
String zsonStr = data.readString();
ZSONObject zson = ZSONObject.stringToZSON(zsonStr);
switch (code) {
case 100: {
//当js发送code为100时，开启语音识别
asrUtils.start();
break;
}
case 101: {
//当js发送code为101时，关闭语音识别
asrUtils.stop();
break;
}
case 200: {
//当js发送code为200时，订阅获取识别结果事件
remoteObjectHandlers.put(200 ,data.readRemoteObject());
//定时获取语音识别结果并返回JS UI
getAsrText();
break;
}
default: {
reply.writeString("service not defined");
return false;
}
}
reply.writeString(ZSONObject.toZSONString(zsonResult));
return true;
}
@Override
public IRemoteObject asObject() {
return this;
}
}
public void getAsrText() {
new Thread(() -> {
while (true) {
try {
Thread.sleep(1 * 500);
Map zsonResult = new HashMap();
zsonResult.put("text",asrUtils.getResultAndClear());
ReportEvent(200, zsonResult);
} catch (RemoteException | InterruptedException e) {
break;
}
}
}).start();
}
private void ReportEvent(int remoteHandler, Object backData) throws RemoteException {
MessageParcel data = MessageParcel.obtain();
MessageParcel reply = MessageParcel.obtain();
MessageOption option = new MessageOption();
data.writeString(ZSONObject.toZSONString(backData));
IRemoteObject remoteObject = remoteObjectHandlers.get(remoteHandler);
remoteObject.sendRequest(100, data, reply, option);
reply.reclaim();
data.reclaim();
}
}