dataload_csv.cpp

Go to the documentation of this file.

#include "dataload_csv.h"
#include <QTextStream>
#include <QFile>
#include <QMessageBox>
#include <QDebug>
#include <QSettings>
#include <QProgressDialog>
#include "selectlistdialog.h"

DataLoadCSV::DataLoadCSV()
{
    _extensions.push_back( "csv");
}

const QRegExp csv_separator("(\\,|\\;|\\|)");

const std::vector<const char*> &DataLoadCSV::compatibleFileExtensions() const
{
    return _extensions;
}

QSize DataLoadCSV::parseHeader(QFile *file, std::vector<std::string>& ordered_names)
{
    QTextStream inA(file);

    QString first_line = inA.readLine();
    QString second_line = inA.readLine();

    QStringList firstline_items = first_line.split(csv_separator);

    int linecount = 0;
    const int columncount = firstline_items.count();

    for (int i=0; i < firstline_items.size(); i++ )
    {
        // remove annoying prefix
        QString field_name ( firstline_items[i] );

        if( field_name.isEmpty())
        {
            field_name = QString("_Column_%1").arg(i);
        }
        ordered_names.push_back( field_name.toStdString() );
    }

    while (!inA.atEnd())
    {
        inA.readLine();
        linecount++;
    }

    QSize table_size;
    table_size.setWidth( columncount);
    table_size.setHeight( linecount );
    return table_size;
}

bool DataLoadCSV::readDataFromFile(FileLoadInfo* info, PlotDataMapRef& plot_data)
{
    bool use_provided_configuration = false;

    if( info->plugin_config.hasChildNodes() )
    {
        use_provided_configuration = true;
        xmlLoadState( info->plugin_config.firstChildElement() );
    }

    const int TIME_INDEX_NOT_DEFINED = -2;

    int time_index = TIME_INDEX_NOT_DEFINED;

    QFile file( info->filename );
    file.open(QFile::ReadOnly);

    std::vector<std::string> column_names;

    const QSize table_size = parseHeader( &file, column_names);
    const int tot_lines   = table_size.height() -1;
    const int columncount = table_size.width();

    file.close();
    file.open(QFile::ReadOnly);
    QTextStream inB( &file );

    std::vector<PlotData*> plots_vector;

    bool interrupted = false;

    int linecount = 0;

    QProgressDialog progress_dialog;
    progress_dialog.setLabelText("Loading... please wait");
    progress_dialog.setWindowModality( Qt::ApplicationModal );
    progress_dialog.setRange(0, tot_lines -1);
    progress_dialog.setAutoClose( true );
    progress_dialog.setAutoReset( true );
    progress_dialog.show();

    // remove first line (header)
    inB.readLine();

    //---- build plots_vector from header  ------
    std::deque<std::string> valid_field_names;

    for (unsigned i=0; i < column_names.size(); i++ )
    {
        const std::string& field_name = ( column_names[i] );

        auto it = plot_data.addNumeric(field_name);

        valid_field_names.push_back( field_name );
        plots_vector.push_back( &(it->second) );

        if (time_index == TIME_INDEX_NOT_DEFINED && use_provided_configuration)
        {
            if( _default_time_axis == field_name )
            {
                time_index = i ;
            }
        }
    }

    if( time_index == TIME_INDEX_NOT_DEFINED && !use_provided_configuration)
    {
        valid_field_names.push_front( "INDEX (auto-generated)" );

        SelectFromListDialog* dialog = new SelectFromListDialog( valid_field_names );
        dialog->setWindowTitle("Select the time axis");
        int res = dialog->exec();

        if (res == QDialog::Rejected )
        {
            return false;
        }

        const int selected_item = dialog->getSelectedRowNumber().at(0);
        if( selected_item > 0)
        {
          for (unsigned i=0; i< column_names.size(); i++)
          {
            if( column_names[i] == valid_field_names[selected_item ] )
            {
              _default_time_axis = column_names[i];
              time_index = selected_item -1;
              break;
            }
          }
        }
    }

    //-----------------
    double prev_time = - std::numeric_limits<double>::max();
    bool monotonic_warning = false;

    while (!inB.atEnd() )
    {
        QString line = inB.readLine();

        QStringList string_items = line.split(csv_separator);
        if( string_items.size() != columncount)
        {
          continue;
        }
        double t = linecount;

        if( time_index >= 0)
        {
            bool is_number = false;
            t = string_items[ time_index ].toDouble(&is_number);

            if( !is_number)
            {
                QMessageBox::StandardButton reply;
                reply = QMessageBox::warning(0, tr("Error reading file"),
                                              tr("One of the timestamps is not a valid number. Abort\n") );

                return false;
            }
            if( t < prev_time )
            {
                QMessageBox::StandardButton reply;
                reply = QMessageBox::warning(0, tr("Error reading file"),
                                              tr("Selected time in not strictly monotonic. Loading will be aborted\n") );

                return false;
            }
            else if (t == prev_time)
            {
                monotonic_warning = true;
            }

            prev_time = t;
        }

        int index = 0;
        for (int i=0; i < string_items.size(); i++ )
        {
            bool is_number = false;
            double y = string_items[i].toDouble(&is_number);
            if( is_number )
            {
                PlotData::Point point( t,y );
                plots_vector[index]->pushBack( point );
            }
            index++;
        }

        if(linecount++ %100 == 0)
        {
            progress_dialog.setValue( linecount );
            QApplication::processEvents();
            if( progress_dialog.wasCanceled() ) {
                interrupted = true;
                break;
            }
        }
    }
    file.close();

    if(interrupted)
    {
        progress_dialog.cancel();
        plot_data.numeric.clear();
    }

    if( monotonic_warning )
    {
        QString message = "Two consecutive samples had the same X value (i.e. time).\n"
                          "Since PlotJuggler makes the assumption that timeseries are strictly monotonic, you "
                          "might experience undefined behaviours.\n\n"
                          "You have been warned...";
        QMessageBox::warning(0, tr("Warning"), message );
    }

    return true;
}

DataLoadCSV::~DataLoadCSV()
{

}

bool DataLoadCSV::xmlSaveState(QDomDocument &doc, QDomElement &parent_element) const
{
    QDomElement elem = doc.createElement("default");
    elem.setAttribute("time_axis", _default_time_axis.c_str() );

    parent_element.appendChild( elem );
    return true;
}

bool DataLoadCSV::xmlLoadState(const QDomElement &parent_element)
{
    QDomElement elem = parent_element.firstChildElement( "default" );
    if( !elem.isNull()    )
    {
        if( elem.hasAttribute("time_axis") )
        {
            _default_time_axis = elem.attribute("time_axis").toStdString();
            return true;
        }
    }
    return false;
}