dataload_parquet.cpp

Go to the documentation of this file.

#include "dataload_parquet.h"
#include <QTextStream>
#include <QFile>
#include <QMessageBox>
#include <QDebug>
#include <QSettings>
#include <QProgressDialog>
#include <QDateTime>
#include <QInputDialog>
#include <QListWidget>
 
DataLoadParquet::DataLoadParquet()
{
  ui = new Ui::DialogParquet();
  _dialog = new QDialog();
 
  ui->setupUi(_dialog);
 
  connect(ui->checkBoxDateFormat, &QCheckBox::toggled, this,
          [=](bool checked) { ui->lineEditDateFormat->setEnabled(checked); });
 
  connect(ui->listWidgetSeries, &QListWidget::currentTextChanged, this,
          [=](QString text) { ui->buttonBox->setEnabled(!text.isEmpty()); });
 
  connect(ui->listWidgetSeries, &QListWidget::doubleClicked, this,
          [=](const QModelIndex&) { _dialog->accept(); });
 
  connect(ui->radioButtonIndex, &QRadioButton::toggled, this, [=](bool checked) {
    ui->buttonBox->setEnabled(checked);
    ui->listWidgetSeries->setEnabled(!checked);
  });
 
  QSettings settings;
 
  bool radio_index_checked =
      settings.value("DataLoadParquet::radioIndexChecked", false).toBool();
  ui->radioButtonIndex->setChecked(radio_index_checked);
 
  bool parse_date_time = settings.value("DataLoadParquet::parseDateTime", false).toBool();
  ui->checkBoxDateFormat->setChecked(parse_date_time);
 
  QString date_format = settings.value("DataLoadParquet::dateFromat", false).toString();
  if (date_format.isEmpty() == false)
  {
    ui->lineEditDateFormat->setText(date_format);
  }
}
 
DataLoadParquet::~DataLoadParquet()
{
  delete ui;
}
 
const std::vector<const char*>& DataLoadParquet::compatibleFileExtensions() const
{
  static std::vector<const char*> extensions = { "parquet" };
  return extensions;
}
 
bool DataLoadParquet::readDataFromFile(FileLoadInfo* info, PlotDataMapRef& plot_data)
{
  using parquet::ConvertedType;
  using parquet::Type;
 
  parquet_reader_ = parquet::ParquetFileReader::OpenFile(info->filename.toStdString());
 
  if (!parquet_reader_)
  {
    return false;
  }
 
  std::shared_ptr<parquet::FileMetaData> file_metadata = parquet_reader_->metadata();
  const auto schema = file_metadata->schema();
  const size_t num_columns = file_metadata->num_columns();
 
  std::vector<parquet::Type::type> column_type;
  std::vector<parquet::ConvertedType::type> converted_column_type;
  std::vector<bool> valid_column(num_columns, true);
 
  for (size_t col = 0; col < num_columns; col++)
  {
    auto column = schema->Column(col);
    auto type = column->physical_type();
    auto converted_type = column->converted_type();
 
    column_type.push_back(type);
    converted_column_type.push_back(converted_type);
 
    valid_column[col] =
        (type == Type::BOOLEAN || type == Type::INT32 || type == Type::INT64 ||
         type == Type::FLOAT || type == Type::DOUBLE);
 
    ui->listWidgetSeries->addItem(QString::fromStdString(column->name()));
  }
 
  {
    QSettings settings;
    if (_default_time_axis.isEmpty())
    {
      _default_time_axis =
          settings.value("DataLoadParquet::prevTimestamp", {}).toString();
    }
 
    if (_default_time_axis.isEmpty() == false)
    {
      auto items = ui->listWidgetSeries->findItems(_default_time_axis, Qt::MatchExactly);
      if (items.size() > 0)
      {
        ui->listWidgetSeries->setCurrentItem(items.front());
      }
    }
  }
 
  int ret = _dialog->exec();
  if (ret != QDialog::Accepted)
  {
    return false;
  }
 
  QString selected_stamp;
 
  if (ui->radioButtonSelect->isChecked())
  {
    auto selected = ui->listWidgetSeries->selectedItems();
    if (selected.size() == 1)
    {
      selected_stamp = selected.front()->text();
    }
  }
 
  QSettings settings;
  settings.setValue("DataLoadParquet::prevTimestamp", selected_stamp);
  settings.setValue("DataLoadParquet::radioIndexChecked",
                    ui->radioButtonIndex->isChecked());
  settings.setValue("DataLoadParquet::parseDateTime",
                    ui->checkBoxDateFormat->isChecked());
  settings.setValue("DataLoadParquet::dateFromat", ui->lineEditDateFormat->text());
 
  //-----------------------------
  // Time to parse
  int timestamp_column = -1;
 
  std::vector<PlotData*> series(num_columns, nullptr);
 
  for (size_t col = 0; col < num_columns; col++)
  {
    auto column = schema->Column(col);
    const std::string& name = column->name();
    if (name == selected_stamp.toStdString())
    {
      timestamp_column = col;
    }
    if (valid_column[col])
    {
      series[col] = &(plot_data.addNumeric(name)->second);
    }
  }
 
  parquet::StreamReader os{ std::move(parquet_reader_) };
 
  std::vector<double> row_values(num_columns, 0.0);
 
  int row = 0;
  while (!os.eof())
  {
    // extract an entire row
    for (size_t col = 0; col < num_columns; col++)
    {
      if (!valid_column[col])
      {
        os.SkipColumns(1);
        continue;
      }
      auto type = column_type[col];
      auto converted_type = converted_column_type[col];
 
      switch (type)
      {
        case Type::BOOLEAN: {
          bool tmp;
          os >> tmp;
          row_values[col] = static_cast<double>(tmp);
          break;
        }
        case Type::INT32:
        case Type::INT64: {
          switch (converted_type)
          {
            case ConvertedType::INT_8: {
              int8_t tmp;
              os >> tmp;
              row_values[col] = static_cast<double>(tmp);
              break;
            }
            case ConvertedType::INT_16: {
              int16_t tmp;
              os >> tmp;
              row_values[col] = static_cast<double>(tmp);
              break;
            }
            case ConvertedType::INT_32: {
              int32_t tmp;
              os >> tmp;
              row_values[col] = static_cast<double>(tmp);
              break;
            }
            case ConvertedType::INT_64: {
              int64_t tmp;
              os >> tmp;
              row_values[col] = static_cast<double>(tmp);
              break;
            }
            case ConvertedType::UINT_8: {
              uint8_t tmp;
              os >> tmp;
              row_values[col] = static_cast<double>(tmp);
              break;
            }
            case ConvertedType::UINT_16: {
              uint16_t tmp;
              os >> tmp;
              row_values[col] = static_cast<double>(tmp);
              break;
            }
            case ConvertedType::UINT_32: {
              uint32_t tmp;
              os >> tmp;
              row_values[col] = static_cast<double>(tmp);
              break;
            }
            case ConvertedType::UINT_64: {
              uint64_t tmp;
              os >> tmp;
              row_values[col] = static_cast<double>(tmp);
              break;
            }
            default: {
              // Fallback in case no converted type is provided
              switch (type)
              {
                case Type::INT32: {
                  int32_t tmp;
                  os >> tmp;
                  row_values[col] = static_cast<double>(tmp);
                  break;
                }
                case Type::INT64: {
                  int64_t tmp;
                  os >> tmp;
                  row_values[col] = static_cast<double>(tmp);
                  break;
                }
              }
            }
          }
          break;
        }
        case Type::FLOAT: {
          float tmp;
          os >> tmp;
          row_values[col] = static_cast<double>(tmp);
          break;
        }
        case Type::DOUBLE: {
          os >> row_values[col];
          break;
        }
        default: {
        }
      }  // end switch
    }  // end for column
 
    os >> parquet::EndRow;
 
    double timestamp = timestamp_column >= 0 ? row_values[timestamp_column] : row;
    row++;
 
    for (size_t col = 0; col < num_columns; col++)
    {
      if (!valid_column[col])
      {
        continue;
      }
 
      if (valid_column[col])
      {
        series[col]->pushBack({ timestamp, row_values[col] });
      }
    }
  }
  return true;
}
 
bool DataLoadParquet::xmlSaveState(QDomDocument& doc, QDomElement& parent_element) const
{
  QDomElement elem = doc.createElement("default");
  elem.setAttribute("prevTimestamp", _default_time_axis);
  elem.setAttribute("radioIndexChecked", ui->radioButtonIndex->isChecked());
  elem.setAttribute("parseDateTime", ui->checkBoxDateFormat->isChecked());
  elem.setAttribute("dateFromat", ui->lineEditDateFormat->text());
 
  parent_element.appendChild(elem);
  return true;
}
 
bool DataLoadParquet::xmlLoadState(const QDomElement& parent_element)
{
  QDomElement elem = parent_element.firstChildElement("default");
  if (!elem.isNull())
  {
    if (elem.hasAttribute("prevTimestamp"))
    {
      _default_time_axis = elem.attribute("prevTimestamp");
    }
    if (elem.hasAttribute("radioIndexChecked"))
    {
      bool checked = elem.attribute("radioIndexChecked").toInt();
      ui->radioButtonIndex->setChecked(checked);
    }
    if (elem.hasAttribute("parseDateTime"))
    {
      bool checked = elem.attribute("parseDateTime").toInt();
      ui->checkBoxDateFormat->setChecked(checked);
    }
    if (elem.hasAttribute("dateFromat"))
    {
      ui->lineEditDateFormat->setText(elem.attribute("dateFromat"));
    }
  }
 
  return true;
}