Vonage Video APIによるリアルタイム翻訳

app.post('/startStreaming', async (req, res) => {
    try {
        console.log('someone wants to stream');
        const {
            streamId,
            sessionId
        } = req.body;
        console.log(streamId, sessionId);
        const response = await startStreamer(streamId, sessionId);
        const render = await createExperienceComposer(process.env.sessionId);
        if (render && response) {
            return res.send(response);
        }
        res.send(response);
    } catch (e) {
        console.log(e);
    }
});

const startStreamer = async (streamId, sessionId) => {
    try {
        const {
            token
        } = await opentok.generateToken(sessionId, 'publisher');


        const data = JSON.stringify({
            sessionId: sessionId,
            token: token,
            websocket: {
                uri: `${process.env.websocket_url}/socket`,
                streams: [streamId],
                headers: {
                    from: streamId,
                },
            },
        });
        const config = {
            method: 'post',
            url: `https://api.opentok.com/v2/project/${process.env.VIDEO_API_API_KEY}/connect`,
            headers: {
                'X-OPENTOK-AUTH': await generateRestToken(),
                'Content-Type': 'application/json',
            },
            data: data,
        };
        const response = await axios(config);
        console.log(response.data);
        return response.data;
    } catch (e) {
        console.log(e);
        return e;
    }
};

app.ws('/socket', async (ws, req) => {
    console.log('someone connected');
    console.log(`starting connection for user with streamId ${streamId}`)
    const uuid = req?.query?.uuid
    let sc = new StreamingClient(uuid, 'es', 'en-US');
    await sc.init();
    const aWss = expressWs.getWss().clients;


    sc.setAudioChunkAvailableCallback(function(chunk) {
        console.log('audio available. Sending audio to the clients connected to the wss');


        const aWss = expressWs.getWss('/socket').clients;
        if (ws.readyState === 1) {
            aWss.forEach(function(client) {
                client.send(chunk);
            });
        }
    });


    sc.setTranscriptionAvailableCallback(function(data) {
        console.log(‘Sending original and translated captions to the connected clients’);
        const aWss = expressWs.getWss('/socket').clients;
        aWss.forEach(function(client) {
            client.send(JSON.stringify(data));
        });
    });
    sc.startRecognizer();
    ws.on('message', (msg) => {
        try {
            if (typeof msg === 'string') {
                let config = JSON.parse(msg);
                console.log(config);
                sc.setId(config.from);
            } else {
                console.log(‘Sending audio data to Google
                    for Speech to text recognition’);
                sc.sendMessage(msg);
            }
        } catch (err) {
            console.log('[' + uuid + '] ' + err);
            ws.removeAllListeners('message');
            ws.close();
        }
    });


    ws.on('close', () => {
        console.log('[' + uuid + '] Websocket closed');
        sc.closeConversation();
        sc = null;
    });
});

app.get('/ec', (req, res) => {
    const options = {
        root: path.join(__dirname),
    };
    res.sendFile('public/ec.html', options);
});

startRecognizer() {
    const config = {
        encoding: encoding,
        sampleRateHertz: sampleRateHertz,
        model: model,
        enableAutomaticPunctuation: true,
        languageCode: this.language,
    };

    const request = {
        config: config,
        interimResults: true,
    };

    this.recognizeStream = this.speechClient.streamingRecognize(request);
    this.recognizeStream.on('data', async (data) => {
        let originalText = data.results\[0].alternatives\[0].transcript;
        console.log('\[' + this.id + ']\[User]' + originalText);
        let translatedText = await this.translate(originalText);
        if (this.transcriptAvailableCallback) {
            this.transcriptAvailableCallback({
                original: originalText,
                translated: translatedText,
                id: this.id,
            });
        }
        this.getAudioData(translatedText);
    });

    this.recognizeStream.on('error', (err) => {
        console.error(err);
        this.tryEndAudioStream();
        if (this.errorCallback) {
            this.errorCallback(err);
        }
    });

    this.recognizeStream.on('finish', (res) => {});
    console.log('\[' + this.id + '] Stream Created');
    this.waitingRestart = false;
    return this.recognizeStream;
}

async translate(text) {
    let output = '';
    try {
        let [translations] = await this.translationClient.translate(text, this.outputLanguage);
        translations = Array.isArray(translations) ? translations : [translations];
        translations.forEach((translation, i) => {
            output += `${translation}`;
        });
    } catch (err) {
        console.log(err);
    }
    console.log(output);
    return output;
}

async getAudioData(text, isSessionEnding = false) {
    try {
        this.sessionEnding = isSessionEnding;
        const request = {
            input: {
                text: text
            },
            voice: {
                languageCode: this.outputLanguage,
                ssmlGender: 'NEUTRAL'
            },
            audioConfig: {
                audioEncoding: encoding,
                sampleRateHertz: sampleRateHertz,
            },
        };
        const [response] = await this.ttsClient.synthesizeSpeech(request);
        if (this.audioChunkAvailableCallback) {
            this.audioChunkAvailableCallback(response.audioContent);
        }
    } catch (err) {
        if (this.errorCallback) {
            this.errorCallback(err);
        }
    }
}