9 #include <QProgressDialog>
11 #include <QInputDialog>
12 #include <QPushButton>
13 #include <QSyntaxStyle>
14 #include <QRadioButton>
19 #include <QStandardItemModel>
23 static constexpr
const char*
INDEX_AS_TIME =
"__TIME_INDEX_GENERATED__";
25 void SplitLine(
const QString& line, QChar separator, QStringList& parts)
28 bool inside_quotes =
false;
29 bool quoted_word =
false;
35 for (
int pos = 0; pos < line.size(); pos++)
46 quote_start = pos + 1;
48 inside_quotes = !inside_quotes;
51 bool part_completed =
false;
52 bool add_empty =
false;
55 if ((!inside_quotes && line[pos] == separator))
57 part_completed =
true;
59 if (pos + 1 == line.size())
61 part_completed =
true;
64 if (line[pos] == separator)
76 part = line.mid(quote_start, quote_end - quote_start + 1);
80 part = line.mid(start_pos, end_pos - start_pos);
83 parts.push_back(part.trimmed());
86 inside_quotes =
false;
90 parts.push_back(QString());
103 _ui =
new Ui::DialogCSV();
108 _ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(
false);
110 connect(
_ui->radioButtonSelect, &QRadioButton::toggled,
this, [
this](
bool checked) {
111 _ui->listWidgetSeries->setEnabled(checked);
112 auto selected = _ui->listWidgetSeries->selectionModel()->selectedIndexes();
113 bool box_enabled = !checked || selected.size() == 1;
114 _ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(box_enabled);
116 connect(
_ui->listWidgetSeries, &QListWidget::itemSelectionChanged,
this, [
this]() {
117 auto selected = _ui->listWidgetSeries->selectionModel()->selectedIndexes();
118 bool box_enabled = _ui->radioButtonIndex->isChecked() || selected.size() == 1;
119 _ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(box_enabled);
122 connect(
_ui->listWidgetSeries, &QListWidget::itemDoubleClicked,
this,
123 [
this]() { emit _ui->buttonBox->accepted(); });
125 connect(
_ui->radioCustomTime, &QRadioButton::toggled,
this,
126 [
this](
bool checked) { _ui->lineEditDateFormat->setEnabled(checked); });
128 connect(
_ui->dateTimeHelpButton, &QPushButton::clicked,
this,
129 [
this]() { _dateTime_dialog->show(); });
132 QSizePolicy sp_retain =
_ui->tableView->sizePolicy();
133 sp_retain.setRetainSizeWhenHidden(
true);
134 _ui->tableView->setSizePolicy(sp_retain);
136 _ui->splitter->setStretchFactor(0, 1);
137 _ui->splitter->setStretchFactor(1, 2);
139 _model =
new QStandardItemModel;
157 file.open(QFile::ReadOnly);
161 column_names.clear();
162 _ui->listWidgetSeries->clear();
164 QTextStream inA(&file);
167 QString first_line = inA.readLine();
169 QString preview_lines = first_line +
"\n";
171 QStringList firstline_items;
174 int is_number_count = 0;
176 std::set<std::string> different_columns;
179 for (
int i = 0; i < firstline_items.size(); i++)
182 firstline_items[i].trimmed().toDouble(&isNum);
189 if (is_number_count == firstline_items.size())
191 for (
int i = 0; i < firstline_items.size(); i++)
193 auto field_name = QString(
"_Column_%1").arg(i);
194 auto column_name = field_name.toStdString();
195 column_names.push_back(column_name);
196 different_columns.insert(column_name);
201 for (
int i = 0; i < firstline_items.size(); i++)
204 QString field_name(firstline_items[i].trimmed());
206 if (field_name.isEmpty())
208 field_name = QString(
"_Column_%1").arg(i);
210 auto column_name = field_name.toStdString();
211 column_names.push_back(column_name);
212 different_columns.insert(column_name);
216 if (different_columns.size() < column_names.size())
220 QMessageBox::warning(
nullptr,
"Duplicate Column Name",
221 "Multiple Columns have the same name.\n"
222 "The column number will be added (as suffix) to the name.");
226 std::vector<size_t> repeated_columns;
227 for (
size_t i = 0; i < column_names.size(); i++)
229 repeated_columns.clear();
230 repeated_columns.push_back(i);
232 for (
size_t j = i + 1; j < column_names.size(); j++)
234 if (column_names[i] == column_names[j])
236 repeated_columns.push_back(j);
239 if (repeated_columns.size() > 1)
241 for (
size_t index : repeated_columns)
243 QString suffix =
"_";
244 suffix += QString::number(index).rightJustified(2,
'0');
245 column_names[index] += suffix.toStdString();
251 QStringList column_labels;
252 for (
const auto&
name : column_names)
254 auto qname = QString::fromStdString(
name);
255 _ui->listWidgetSeries->addItem(qname);
256 column_labels.push_back(qname);
258 _model->setColumnCount(column_labels.size());
259 _model->setHorizontalHeaderLabels(column_labels);
263 for (
int row = 0; row <= 100 && !inA.atEnd(); row++)
265 auto line = inA.readLine();
266 preview_lines += line +
"\n";
267 lines.push_back(line);
270 _model->setRowCount(lines.count());
271 for (
int row = 0; row < lines.count(); row++)
274 for (
int j = 0; j < lineToken.size(); j++)
276 QString value = lineToken[j].toString();
277 if (
auto item =
_model->item(row, j))
279 item->setText(value);
283 _model->setItem(row, j,
new QStandardItem(value));
288 _ui->rawText->setPlainText(preview_lines);
289 _ui->tableView->resizeColumnsToContents();
296 column_names->clear();
297 _ui->tabWidget->setCurrentIndex(0);
300 _dialog->restoreGeometry(settings.value(
"DataLoadCSV.geometry").toByteArray());
302 _ui->radioButtonIndex->setChecked(
303 settings.value(
"DataLoadCSV.useIndex",
false).toBool());
304 bool use_custom_time = settings.value(
"DataLoadCSV.useDateFormat",
false).toBool();
307 _ui->radioCustomTime->setChecked(
true);
311 _ui->radioAutoTime->setChecked(
true);
313 _ui->lineEditDateFormat->setText(
314 settings.value(
"DataLoadCSV.dateFormat",
"yyyy-MM-dd hh:mm:ss").toString());
318 file.open(QFile::ReadOnly);
319 QTextStream
in(&file);
321 QString first_line =
in.readLine();
322 int comma_count = first_line.count(QLatin1Char(
','));
323 int semicolon_count = first_line.count(QLatin1Char(
';'));
324 int space_count = first_line.count(QLatin1Char(
' '));
325 int tab_count = first_line.count(QLatin1Char(
'\t'));
327 if (comma_count > 3 && comma_count > semicolon_count)
329 _ui->comboBox->setCurrentIndex(0);
332 if (semicolon_count > 3 && semicolon_count > comma_count)
334 _ui->comboBox->setCurrentIndex(1);
337 if (space_count > 3 && comma_count == 0 && semicolon_count == 0)
339 _ui->comboBox->setCurrentIndex(2);
342 if (tab_count > 3 && comma_count == 0 && semicolon_count == 0)
344 _ui->comboBox->setCurrentIndex(3);
350 QString theme = settings.value(
"StyleSheet::theme",
"light").toString();
351 auto style_path = (theme ==
"light") ?
":/resources/lua_style_light.xml" :
352 ":/resources/lua_style_dark.xml";
354 QFile fl(style_path);
355 if (fl.open(QIODevice::ReadOnly))
358 if (style->load(fl.readAll()))
360 _ui->rawText->setSyntaxStyle(style);
365 std::unique_ptr<QObject> pcontext(
new QObject);
366 QObject* context = pcontext.get();
367 QObject::connect(
_ui->comboBox, qOverload<int>(&QComboBox::currentIndexChanged),
368 context, [&](
int index) {
369 const std::array<char, 4> delimiters = {
',',
';',
' ',
'\t' };
370 _delimiter = delimiters[std::clamp(index, 0, 3)];
376 parseHeader(file, *column_names);
378 QString previous_index = settings.value(
"DataLoadCSV.timeIndex",
"").toString();
379 if (previous_index.isEmpty() ==
false)
381 auto items = _ui->listWidgetSeries->findItems(previous_index, Qt::MatchExactly);
382 if (items.size() > 0)
384 _ui->listWidgetSeries->setCurrentItem(items.front());
388 int res = _dialog->exec();
390 settings.setValue(
"DataLoadCSV.geometry", _dialog->saveGeometry());
391 settings.setValue(
"DataLoadCSV.useIndex", _ui->radioButtonIndex->isChecked());
392 settings.setValue(
"DataLoadCSV.useDateFormat", _ui->radioCustomTime->isChecked());
393 settings.setValue(
"DataLoadCSV.dateFormat", _ui->lineEditDateFormat->text());
395 if (res == QDialog::Rejected)
400 if (_ui->radioButtonIndex->isChecked())
405 QModelIndexList indexes = _ui->listWidgetSeries->selectionModel()->selectedRows();
406 if (indexes.size() == 1)
408 int row = indexes.front().row();
409 auto item = _ui->listWidgetSeries->item(row);
410 settings.setValue(
"DataLoadCSV.timeIndex", item->text());
419 bool is_number =
false;
420 QString str_trimmed = str.trimmed();
424 int64_t ts = str.toLong(&is_number);
425 const int64_t first_ts = 1400000000;
426 const int64_t last_ts = 2000000000;
431 if (ts > first_ts * 1e9 && ts < last_ts * 1e9)
433 val = double(ts) * 1e-9;
435 else if (ts > first_ts * 1e6 && ts < last_ts * 1e6)
439 val = double(ts) * 1e-6;
449 val = str.toDouble(&is_number);
455 static QLocale locale_with_comma(QLocale::German);
456 val = locale_with_comma.toDouble(str, &is_number);
460 QDateTime ts = QDateTime::fromString(str, Qt::ISODateWithMs);
463 return double(ts.toMSecsSinceEpoch()) / 1000.0;
470 return is_number ? std::optional<double>(val) :
std::nullopt;
475 QDateTime ts = QDateTime::fromString(str,
format);
478 return double(ts.toMSecsSinceEpoch()) / 1000.0;
490 std::vector<std::string> column_names;
508 for (
size_t i = 0; i < column_names.size(); i++)
525 bool interrupted =
false;
531 file.open(QFile::ReadOnly);
532 QTextStream
in(&file);
541 QProgressDialog progress_dialog;
542 progress_dialog.setWindowTitle(
"Loading the CSV file");
543 progress_dialog.setLabelText(
"Loading... please wait");
544 progress_dialog.setWindowModality(Qt::ApplicationModal);
545 progress_dialog.setRange(0, tot_lines - 1);
546 progress_dialog.setAutoClose(
true);
547 progress_dialog.setAutoReset(
true);
548 progress_dialog.show();
552 std::vector<PlotData*> plots_vector;
553 std::vector<StringSeries*> string_vector;
554 bool sortRequired =
false;
556 for (
unsigned i = 0; i < column_names.size(); i++)
558 const std::string& field_name = (column_names[i]);
559 auto num_it = plot_data.
addNumeric(field_name);
560 plots_vector.push_back(&(num_it->second));
563 string_vector.push_back(&(str_it->second));
567 double prev_time = std::numeric_limits<double>::lowest();
569 const bool parse_date_format =
_ui->radioCustomTime->isChecked();
571 auto ParseNumber = [&](QString str,
bool& is_number) {
572 QString str_trimmed = str.trimmed();
573 double val = val = str_trimmed.toDouble(&is_number);
577 static QLocale locale_with_comma(QLocale::German);
578 val = locale_with_comma.toDouble(str_trimmed, &is_number);
583 if (parse_date_format)
589 ts = QDateTime::fromString(str_trimmed, Qt::ISODateWithMs);
591 is_number = ts.isValid();
594 val = ts.toMSecsSinceEpoch() / 1000.0;
600 file.open(QFile::ReadOnly);
601 QTextStream
in(&file);
603 QString header_str =
in.readLine();
604 QStringList string_items;
605 QStringList header_string_items;
608 QString time_header_str;
614 QString line =
in.readLine();
618 if (string_items.size() == 0)
623 if (string_items.size() != column_names.size())
626 msgBox.setWindowTitle(tr(
"Error reading file"));
627 msgBox.setText(tr(
"The number of values at line %1 is %2,\n"
628 "but the expected number of columns is %3.\n"
631 .
arg(string_items.size())
632 .arg(column_names.size()));
634 msgBox.setDetailedText(tr(
"File: \"%1\" \n\n"
635 "Error reading file | Mismatched field count\n"
637 "Header fields: %6\n"
638 "Fields on line [%4]: %7\n\n"
648 .arg(column_names.size())
649 .arg(string_items.size()));
651 QPushButton* abortButton = msgBox.addButton(QMessageBox::Ok);
653 msgBox.setIcon(QMessageBox::Warning);
660 double timestamp = linecount;
664 t_str = string_items[time_index];
665 const auto time_trimm = t_str.trimmed();
666 bool is_number =
false;
667 if (parse_date_format)
684 time_header_str = header_string_items[time_index];
689 msgBox.setWindowTitle(tr(
"Error reading file"));
690 msgBox.setText(tr(
"Couldn't parse timestamp on line %1 with string \"%2\" . "
695 msgBox.setDetailedText(tr(
"File: \"%1\" \n\n"
696 "Error reading file | Couldn't parse timestamp\n"
697 "Parsing format: [%4]\n"
698 "Time at line %2 : [%3]\n")
706 QPushButton* abortButton = msgBox.addButton(QMessageBox::Ok);
708 msgBox.setIcon(QMessageBox::Warning);
715 if (prev_time > timestamp && !sortRequired)
719 timeName = time_header_str;
721 msgBox.setWindowTitle(tr(
"Selected time is not monotonic"));
722 msgBox.setText(tr(
"PlotJuggler detected that the time in this file is "
723 "non-monotonic. This may indicate an issue with the input "
724 "data. Continue? (Input file will not be modified but data "
725 "will be sorted by PlotJuggler)"));
726 msgBox.setDetailedText(tr(
"File: \"%1\" \n\n"
727 "Selected time is not monotonic\n"
728 "Time Index: %6 [%7]\n"
729 "Time at line %2 : %3\n"
730 "Time at line %4 : %5")
739 QPushButton* sortButton =
740 msgBox.addButton(tr(
"Continue"), QMessageBox::ActionRole);
741 QPushButton* abortButton = msgBox.addButton(QMessageBox::Abort);
742 msgBox.setIcon(QMessageBox::Warning);
745 if (msgBox.clickedButton() == abortButton)
749 else if (msgBox.clickedButton() == sortButton)
759 prev_time = timestamp;
763 for (
unsigned i = 0; i < string_items.size(); i++)
765 bool is_number =
false;
766 const auto& str = string_items[i];
767 double y = ParseNumber(str, is_number);
770 plots_vector[i]->pushBack({ timestamp,
y });
774 string_vector[i]->pushBack({ timestamp, str.toStdString() });
778 if (linecount++ % 100 == 0)
780 progress_dialog.setValue(linecount);
781 QApplication::processEvents();
782 if (progress_dialog.wasCanceled())
792 progress_dialog.cancel();
807 for (
unsigned i = 0; i < column_names.size(); i++)
809 const auto&
name = column_names[i];
810 bool is_numeric =
true;
811 if (plots_vector[i]->
size() == 0 && string_vector[i]->
size() > 0)
829 QDomElement elem = doc.createElement(
"parameters");
831 elem.setAttribute(
"delimiter",
_ui->comboBox->currentIndex());
834 if (
_ui->radioCustomTime->isChecked())
836 elem.setAttribute(
"date_format",
_ui->lineEditDateFormat->text());
839 parent_element.appendChild(elem);
845 QDomElement elem = parent_element.firstChildElement(
"parameters");
850 if (elem.hasAttribute(
"time_axis"))
854 if (elem.hasAttribute(
"delimiter"))
856 int separator_index = elem.attribute(
"delimiter").toInt();
857 _ui->comboBox->setCurrentIndex(separator_index);
858 switch (separator_index)
871 if (elem.hasAttribute(
"date_format"))
873 _ui->radioCustomTime->setChecked(
true);
874 _ui->lineEditDateFormat->setText(elem.attribute(
"date_format"));
878 _ui->radioAutoTime->setChecked(
true);