audioinput.cpp

Go to the documentation of this file.

/****************************************************************************
**
** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
** All rights reserved.
** Contact: Nokia Corporation (qt-info@nokia.com)
**
** This file is part of the examples of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial Usage
** Licensees holding valid Qt Commercial licenses may use this file in
** accordance with the Qt Commercial License Agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Nokia.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Nokia gives you certain additional
** rights.  These rights are described in the Nokia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file.  Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
** If you have questions regarding the use of this file, please contact
** Nokia at qt-info@nokia.com.
** $QT_END_LICENSE$
**
****************************************************************************/

#include <stdlib.h>
#include <math.h>

#include <QDebug>
#include <QPainter>
#include <QPalette>
#include <QVBoxLayout>

#include <QAudioDeviceInfo>
#include <QAudioInput>
#include "audioinput.h"
#include "wavefile.h"

ATRASR_CLIENT _atrasrClient;
int _epdPort = 0;

static int _logCounter = 0;
static int _speakCount = 0;
static int _filenum = 1;

static int _epdOnFrameNo = -1;
static int _epdOffFrameNo = -1;

#define BUFFER_SIZE 4096

AudioInfo::AudioInfo(QObject* parent, QAudioInput* device, SoundServer *server)
    :QIODevice( parent )
{
    input = device;
    m_server = server;

    m_bRec = false;
    m_file = NULL;

    m_bEpdOn = true; //(_atrasrClient.socket != NULL);
    m_bEpdEnabled = m_bEpdOn;
    if (_atrasrClient.socket != NULL){
        connect(_atrasrClient.socket, SIGNAL(readyRead()), this, SLOT(epdRead()));
    }
    connect(server, SIGNAL(onCmd(QString)), this, SLOT(onCmd(QString)));
    connect(server, SIGNAL(clientConnected(QHostAddress)), this, SLOT(onClientConnected(QHostAddress)));
}

AudioInfo::~AudioInfo()
{
}

QList<AudioInfo::Level> AudioInfo::getLevels()
{
    QList<Level> levels = m_levels;
    m_levels.clear();
    return levels;
}

bool AudioInfo::epdEnabled() const
{
    return m_bEpdEnabled;
}

void AudioInfo::setEpdEnabled(bool bEnabled)
{
    m_bEpdEnabled = bEnabled;
    m_bEpdOn = bEnabled;

    if ( bEnabled ) {
        m_server->SendEpdOn();
    } else {
        m_server->SendEpdOff();
        recstop(false);
    }
}

void AudioInfo::start()
{
    open(QIODevice::WriteOnly);
}

void AudioInfo::stop()
{
    recstop();
    close();
}

qint64 AudioInfo::readData(char *data, qint64 maxlen)
{
    Q_UNUSED(data)
    Q_UNUSED(maxlen)

    return 0;
}

qint64 AudioInfo::writeData(const char *data, qint64 len)
{
    if (_atrasrClient.socket == NULL){
        processWave(data, len);
    }
    else{
        AtrasrSendDATA(&_atrasrClient, data, len);
    }

    return len;
}

void AudioInfo::processWave(const void *data, qint64 len)
{
    if (m_bRec){
        if (m_file != NULL){
            m_file->write((const char*)data,len);
        }
        m_server->SendData(data,len);
        m_sampleSize += len;
    }

    // sample format is S16LE, only!
    int min = 0;
    int max = 0;

    int samples = len/2; // 2 bytes per sample
    qint16* s = (qint16*)data;
    for(int i = 0; i < samples; i++) {
        qint16 sample = *s;
        s++;
        if(sample < min) min = sample;
        if(sample > max) max = sample;

        if (i > 0 && i % 480 == 0){
            m_levels.append(Level(min,max,m_bRec,m_bEpdOn));
            min = 0;
            max = 0;
        }
    }
    m_levels.append(Level(min,max,m_bRec,m_bEpdOn));

    emit update();
}

void AudioInfo::epdRead()
{
    long frameType;
    int frameIndex;
    unsigned char *pData;
    static unsigned char recvData[ATRASR_RECV_BUF_FRAME_COUNT * ATRASR_FRAME_DATA_SIZE];
    int recvSize = 0;

    int frameCount = AtrasrReceiveFrame( &_atrasrClient );

    for ( frameIndex = 0; frameIndex < frameCount; frameIndex++ )
    {
        pData = AtrasrParseFrame( &_atrasrClient, frameIndex, &frameType );

        if ( frameType == ATRASR_TOF ) {
            qDebug() << "NICTASR : TOF frame received.";

            if ( m_bEpdOn ) {
                m_server->SendEpdOn();
            } else {
                m_server->SendEpdOff();
            }
        }
        else if ( frameType == ATRASR_DATA ) {
            memcpy( recvData + recvSize, pData, ATRASR_FRAME_DATA_SIZE );
            recvSize += ATRASR_FRAME_DATA_SIZE;

            if(_logCounter % 80 == 0){
                _logCounter = 0;
                qDebug() << "send : " << _atrasrClient.sendFrameCount << "recv : " << _atrasrClient.recvFrameCount << "Off : " << _epdOffFrameNo << "On : " << _epdOnFrameNo;
            }
            _logCounter++;

            /* EPDのON切り替え */
            if (_epdOnFrameNo >= 0 && _atrasrClient.sendFrameCount >= _epdOnFrameNo){
                qDebug() << "EPD ON.";
                _speakCount--;
                if(_speakCount<=0){
                    _speakCount=0;
                    m_bEpdOn = true;
                    emit validateRecButton(true);
                    m_server->SendEpdOn();
                }
                _epdOnFrameNo = -1;
            }

            if (_epdOffFrameNo >= 0 &&_atrasrClient.sendFrameCount >= _epdOffFrameNo ) {
                qDebug() << "EPD OFF.";
                m_bEpdOn = false;
                m_server->SendEpdOff();
                _epdOffFrameNo = -1;
            }
        }
        else if ( frameType == ATRASR_EOF ) {
            qDebug() << "NICTASR : EOF frame received.";
            qDebug() << "NICTASR : " << _atrasrClient.sendFrameCount << " frames sent. " << _atrasrClient.recvFrameCount << " frames received";
        }
        else if ( frameType == ATRASR_START ) {
            if ( recvSize > 0 ) {
                qDebug() << "RECVSIZE : " << recvSize;
                processWave( recvData, recvSize );
                recvSize = 0;
            }

            qDebug() << "NICTASR : START frame received. (" << _atrasrClient.sendFrameCount << " sent, " << _atrasrClient.recvFrameCount << " received)";

            if ( m_bEpdOn ){
                recstart();
            }
        }
        else if ( frameType == ATRASR_END || frameType == ATRASR_CANCEL ) {
            if ( recvSize > 0 ) {
                processWave( recvData, recvSize );
                recvSize = 0;
            }

            qDebug() << "NICTASR : " << (( frameType == ATRASR_END ) ? "END" : "CANCEL") << " frame received. ";

            if ( m_bEpdOn ){
                recstop( frameType == ATRASR_END );
            }
        }
        else {
            qDebug() << "NICTASR : Unknown frame type : " << frameType;
        }
    }

    if ( recvSize > 0 ) {
        processWave( recvData, recvSize );
    }
}

void AudioInfo::onCmd(const QString &cmd)
{
    qDebug() << "Command Received - " << cmd;

    if (cmd.isEmpty()){
        return;
    }

    // cancel recording.
    if (m_bRec){
        recstop(false);
    }

    // Epd ON/OFF
    if ( cmd == "epdon" || cmd == "epdoff" ){
        /* ATRASRと繋いでいるときだけ有効 */
        if ( _atrasrClient.socket && m_bEpdEnabled ){
            /* 音声採取時のフレーム番号(+オフセット）を覚えておき、それに対してEPDのONを行う。 */
            const int OFFSET = 80;
            if ( cmd == "epdon" ){
                _epdOnFrameNo = _atrasrClient.sendFrameCount;
                _epdOnFrameNo += OFFSET;
            }
            /* OFFはすぐに行う。前回のONフレームを待っている状態なら取り消す（0FF優先）。 */
            else{
                _epdOffFrameNo = _atrasrClient.sendFrameCount;
                _speakCount++;
                emit validateRecButton(false);
                m_bEpdOn = false;
            }

            qDebug() << "Send frame count:" << _atrasrClient.sendFrameCount << " Recv frame count: " << _atrasrClient.recvFrameCount;
        }
        else{
            qDebug() << "Invalid command in non-auto-detect mode : " << cmd;
        }
    }
    else{
        qDebug() << "Unknown command - " << cmd;
    }
}

void AudioInfo::onClientConnected(const QHostAddress &addr)
{
    if (_epdPort == 0 || _atrasrClient.socket != NULL){
        return;
    }

    if ( AtrasrConnect( &_atrasrClient, addr.toString().toLatin1(), _epdPort ) != -1 ){
        AtrasrSendTOF( &_atrasrClient );
        AtrasrSendSTARTPU( &_atrasrClient );
        connect(_atrasrClient.socket, SIGNAL(readyRead()), this, SLOT(epdRead()));
        connect(_atrasrClient.socket, SIGNAL(disconnected()), this, SLOT(epdDisconnected()));
    }
}

void AudioInfo::epdDisconnected()
{
    _atrasrClient.socket->deleteLater();
    _atrasrClient.socket = NULL;

    _speakCount = 0;
    _epdOnFrameNo = -1;
    if (m_bEpdEnabled){
        m_bEpdOn = true;
    }

    recstop(false);
}

void AudioInfo::recstart()
{
    m_bRec = true;

    QString fileSufix;
    if (!m_filename.isEmpty()){
        fileSufix.sprintf(".%04d", _filenum++);
        m_file = WaveFile::createHeader(m_filename + fileSufix);
    }
    m_sampleSize = 0;

    m_server->SendStart();
}

void AudioInfo::recstop(bool bEnd)
{
    if (m_bRec){
        m_bRec = false;
        if (m_file != NULL){
            WaveFile::fillHeader(m_file, m_sampleSize);
            QString filename = m_file->fileName();
            m_file->close();
            delete m_file;
            m_file = NULL;
            QFile::remove(filename+".wav");
            QFile::rename(filename, filename+".wav");
        }
        if (bEnd){
            m_server->SendEnd();
        }
        else{
            m_server->SendEnd();
            //m_server->SendCancel();
        }
    }
}

QString AudioInfo::m_filename;

void AudioInfo::setFileName(const QString &filename)
{
    m_filename = filename;
}

int RenderArea::STEP = 150;

RenderArea::RenderArea(QWidget *parent)
    : QWidget(parent)
{
    //setBackgroundRole(QPalette::Base);
    //setAutoFillBackground(true);

    setMinimumHeight(30);
    setMinimumWidth(50);

    for(int i = 0; i < STEP; i++){
        m_levels.enqueue(AudioInfo::Level());
    }
}

void RenderArea::paintEvent(QPaintEvent * /* event */)
{
    QPainter painter(this);

    int centerY = painter.viewport().center().y();

    for(int i = 0; i < STEP; i++){
        AudioInfo::Level level = m_levels.at(i);

        QRect rect(
            (double)painter.viewport().left() + i*(double)painter.viewport().width()/STEP,
            painter.viewport().top(),
            (double)painter.viewport().width()/STEP,
            painter.viewport().height()
        );
        QColor color = !level.epdOn ? Qt::lightGray :(level.recording ? Qt::blue : Qt::black);
        painter.setPen(color);
        painter.setBrush(QBrush(color, Qt::SolidPattern));
        painter.drawRect(rect);

        rect.setTop( centerY - level.max*painter.viewport().height()/2 );
        rect.setHeight( (level.max-level.min)*painter.viewport().height()/2);

        color = level.recording ? Qt::yellow : QColor(127, 255, 255);
        painter.setPen(color);
        painter.setBrush(QBrush(color, Qt::SolidPattern));
        painter.drawRect(rect);
    }
}

void RenderArea::setLevel(const QList<AudioInfo::Level> &levels)
{
    m_levels.append(levels);
    while (m_levels.count() > STEP){
        m_levels.dequeue();
    }
    repaint();
}


SoundServerApp::SoundServerApp(SoundServer *_server)
    : server(_server)
{
    QWidget *window = new QWidget;
    QVBoxLayout* layout = new QVBoxLayout;

    canvas = new RenderArea;
    layout->addWidget(canvas);

    recButton = new QPushButton(this);
    recButton->setText(tr("REC"));
    recButton->setSizePolicy(QSizePolicy::Minimum, QSizePolicy::Minimum);
    layout->addWidget(recButton);

    deviceBox = new QComboBox(this);
    QList<QAudioDeviceInfo> devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput);
    for(int i = 0; i < devices.size(); ++i) {
        QString deviceName = devices.at(i).deviceName();
        if (deviceName.length() > 20) {
            deviceName = deviceName.left(19).append("...");
        }
        deviceBox->addItem(deviceName, qVariantFromValue(devices.at(i)));
    }
    connect(deviceBox,SIGNAL(activated(int)),SLOT(deviceChanged(int)));
    layout->addWidget(deviceBox);

    window->setLayout(layout);
    setCentralWidget(window);
    window->show();

    buffer = new char[BUFFER_SIZE];

    pullMode = true;

    // AudioInfo class only supports mono S16LE samples!
    format.setSampleRate(WaveFile::SAMPLING_RATE);
    format.setChannelCount(1);
    format.setSampleSize(WaveFile::SAMPLE_BYTE*8);
    format.setSampleType(QAudioFormat::SignedInt);
    format.setByteOrder(QAudioFormat::LittleEndian);
    format.setCodec("audio/pcm");

    audioInput = new QAudioInput(format,this);
    connect(audioInput,SIGNAL(notify()),SLOT(status()));
    connect(audioInput,SIGNAL(stateChanged(QAudio::State)),SLOT(state(QAudio::State)));
    audioinfo  = new AudioInfo(this,audioInput,server);
    connect(audioinfo,SIGNAL(update()),SLOT(refreshDisplay()));
    connect(audioinfo,SIGNAL(validateRecButton(bool)),this,SLOT(epdSpeechChanged(bool)));
    audioinfo->start();
    audioInput->start(audioinfo);

    if (!devices.empty()){
        deviceChanged(0);
    }

    //if (_atrasrClient.socket != NULL){
    if (_epdPort > 0){
        recButton->setCheckable(true);
        recButton->setChecked(true);
        epdEnableChanged(true);
        connect(recButton,SIGNAL(toggled(bool)),this,SLOT(epdEnableChanged(bool)));
    }
    else{
        connect(recButton,SIGNAL(pressed()),audioinfo,SLOT(recstart()));
        connect(recButton,SIGNAL(released()),audioinfo,SLOT(recstop()));
    }
}

SoundServerApp::~SoundServerApp() {}

void SoundServerApp::status()
{
    //qWarning()<<"bytesReady = "<<audioInput->bytesReady()<<" bytes, elapsedUSecs = "<FFFFFFFFF<audioInput->elapsedUSecs()<<", processedUSecs = "<<audioInput->processedUSecs();
}

void SoundServerApp::readMore()
{
    if(!audioInput)
        return;
    qint64 len = audioInput->bytesReady();
    if(len > 4096)
        len = 4096;
    qint64 l = input->read(buffer,len);
    if(l > 0) {
        audioinfo->write(buffer,l);
    }
}

void SoundServerApp::toggleMode()
{
    // Change bewteen pull and push modes
    audioInput->stop();

    if (pullMode) {
        button->setText(tr("Click for Pull Mode"));
        input = audioInput->start();
        connect(input,SIGNAL(readyRead()),SLOT(readMore()));
        pullMode = false;
    } else {
        button->setText(tr("Click for Push Mode"));
        pullMode = true;
        audioInput->start(audioinfo);
    }
}

void SoundServerApp::toggleSuspend()
{
    // toggle suspend/resume
    if(audioInput->state() == QAudio::SuspendedState) {
        qWarning()<<"status: Suspended, resume()";
        audioInput->resume();
        button2->setText("Click To Suspend");
    } else if (audioInput->state() == QAudio::ActiveState) {
        qWarning()<<"status: Active, suspend()";
        audioInput->suspend();
        button2->setText("Click To Resume");
    } else if (audioInput->state() == QAudio::StoppedState) {
        qWarning()<<"status: Stopped, resume()";
        audioInput->resume();
        button2->setText("Click To Suspend");
    } else if (audioInput->state() == QAudio::IdleState) {
        qWarning()<<"status: IdleState";
    }
}

void SoundServerApp::state(QAudio::State state)
{
    qWarning()<<" state="<<state;
}

void SoundServerApp::refreshDisplay()
{
    canvas->setLevel(audioinfo->getLevels());
    canvas->repaint();
}

void SoundServerApp::deviceChanged(int idx)
{
    audioinfo->stop();
    audioInput->stop();
    audioInput->disconnect(this);
    delete audioInput;

    device = deviceBox->itemData(idx).value<QAudioDeviceInfo>();
    audioInput = new QAudioInput(device, format, this);
    connect(audioInput,SIGNAL(notify()),SLOT(status()));
    connect(audioInput,SIGNAL(stateChanged(QAudio::State)),SLOT(state(QAudio::State)));
    audioinfo->start();
    audioInput->start(audioinfo);
}

void SoundServerApp::epdEnableChanged(bool bEnabled)
{
    recButton->setText(bEnabled ? "Speech Detection ON" : "Speech Detection OFF");
    audioinfo->setEpdEnabled(bEnabled);
}

void SoundServerApp::epdSpeechChanged(bool bEnabled)
{
    recButton->setEnabled(bEnabled);
}

void SoundServerApp::recstart()
{
    audioinfo->recstart();
}