diff --git a/README.md b/README.md index c907b67..0de39e2 100644 --- a/README.md +++ b/README.md @@ -1188,3 +1188,168 @@ signal syntax and payload bounds, `VAL_` syntax and reference validation, raw description lookup and preservation through both builders, statement dispatch, signed and unsigned 64-bit boundaries, physical conversion, numeric output, and invalid signal payloads. + +## Measuring code coverage + +Coverage instrumentation is separate from the QtTest suite named `CoverageTest`: +that suite runs extra regression checks, but coverage counters require a specially +instrumented build. Use a separate build directory for each coverage tool. + +### GCC, gcov, and LCOV on Debian/Ubuntu + +Install the build and report tools: + +```sh +sudo apt install build-essential qt6-base-dev qt6-base-dev-tools qmake6 lcov +# Optional IDE: +sudo apt install qtcreator +``` + +GCC supplies `gcov`; the `lcov` package supplies `lcov` and `genhtml`. Use the +`gcov` version matching the GCC used by the selected Qt Creator kit. These +instructions use GCC and Qt 6. + +From the repository root, create a fresh coverage build. `--coverage` must be +passed to both compilation and linking; Debug avoids optimization that can make +source-level results harder to interpret. `-fprofile-abs-path` makes source paths +in the coverage data independent of the compilation working directory. + +```sh +DBC_SOURCE_DIR="$(pwd)" +mkdir -p build-coverage +cd build-coverage + +qmake6 "$DBC_SOURCE_DIR/dbc.pro" \ + CONFIG+=dbc_qt CONFIG+=debug CONFIG-=release \ + 'QMAKE_CXXFLAGS+=--coverage -fprofile-abs-path' \ + 'QMAKE_LFLAGS+=--coverage' +make -j4 + +# The standalone tree test is not a qmake target; instrument it separately. +mkdir -p standalone +c++ -std=c++17 -O0 -g --coverage -fprofile-abs-path \ + -I"$DBC_SOURCE_DIR/core" \ + "$DBC_SOURCE_DIR/tests/tree_node_test.cpp" \ + "$DBC_SOURCE_DIR/core/tree_node.cpp" \ + "$DBC_SOURCE_DIR/core/dbc_tree_builder.cpp" \ + -o standalone/tree_node_tests + +# Capture a zero-count baseline so unexecuted compiled code remains in the report. +lcov --capture --initial --directory . --branch-coverage \ + --include "$DBC_SOURCE_DIR/core/*" --include "$DBC_SOURCE_DIR/qt/*" \ + --output-file baseline.info + +# Reset counts before each new measurement; otherwise executions accumulate. +lcov --zerocounters --directory . +make check +./standalone/tree_node_tests + +lcov --capture --directory . --branch-coverage \ + --include "$DBC_SOURCE_DIR/core/*" --include "$DBC_SOURCE_DIR/qt/*" \ + --output-file executed.info +lcov --add-tracefile baseline.info --add-tracefile executed.info \ + --branch-coverage --output-file coverage.info + +genhtml coverage.info --branch-coverage --demangle-cpp --output-directory html +lcov --summary coverage.info --branch-coverage +``` + +Open `build-coverage/html/index.html` in a browser. The report includes the +library's `core/` and `qt/` sources and instantiated header code; test sources, +generated moc files, and Qt/STL headers are excluded. Click a filename to inspect +execution counts, uncovered lines, and branch outcomes. Branch coverage is a +separate metric: executing a line does not necessarily exercise every outcome. +C++ branch data can also include compiler-generated exception paths. + +For another run with unchanged sources, repeat the reset, test execution, +capture, merge, and HTML generation steps. After changing sources or compiler +flags, rebuild and regenerate `baseline.info` before measuring again. A fresh +build directory avoids mixing old profiling data with a different build. + +An initial measurement with GCC 14.2 and Qt 6.8.2 on 2026-10-07, including both +QtTest executables and the standalone tree test, yielded 97.8% line coverage +(724/740) and 70.7% branch coverage (833/1178). These are a snapshot, not fixed +expected values for later changes or other toolchains. + +See [GCC instrumentation options](https://gcc.gnu.org/onlinedocs/gcc/Instrumentation-Options.html) +and the [LCOV project documentation](https://github.com/linux-test-project/lcov). + +### Configure the GCC coverage build in Qt Creator + +1. Open `dbc.pro` and select a GCC kit with Qt 6. +2. In **Projects > Build Settings**, clone **Debug** as **Coverage**. +3. Choose a separate build directory, such as `build-coverage`. +4. Expand the **qmake** build step and add these **Additional arguments**: + + ```text + CONFIG+=dbc_qt CONFIG+=debug CONFIG-=release "QMAKE_CXXFLAGS+=--coverage -fprofile-abs-path" "QMAKE_LFLAGS+=--coverage" + ``` + +5. Run qmake and build the project. Both `dbc_tests` and `dbc_model_tests` must + run to include coverage from the core and optional model adapter. +6. In a terminal in the coverage build directory, compile the standalone test + and run the LCOV/report commands above. Set `DBC_SOURCE_DIR` to the absolute + repository path if using a new terminal. `make check` runs both QtTest + executables; the standalone tree executable must be run separately. +7. View `html/index.html` in a browser. This workflow produces HTML reports; + the Coco plugin described below uses a different instrumentation toolchain. + +### Coco integration in Qt Creator + +Coco is a separate licensed toolchain. The Qt Creator plugin alone does not +install CoverageScanner, CoverageBrowser, or activate a Coco license. Obtain a +Coco license or evaluation license and download the Linux installer from +[Qt Account downloads](https://account.qt.io/s/downloads). + +For this project's Debian/Ubuntu build, install the compiler, Qt development +tools, and IDE before installing Coco: + +```sh +sudo apt install build-essential qt6-base-dev qt6-base-dev-tools qmake6 qtcreator +# Coco's installer uses locate to discover compilers on Unix: +sudo apt install plocate +sudo updatedb +``` + +Install Coco using the actual downloaded filename, for example: + +```sh +bash /path/to/SquishCocoSetup_VERSION_Linux_x86_64.run +``` + +The exact filename and supported Linux package depend on the downloaded release. +Coco's own tools come from this installer; `lcov` is not required for Coco. +Installing or configuring a new compiler later requires rerunning the installer +or using `cocosetup` to register the compiler. + +For Coco 7.4 or later, set up and check the Qt License Service using the installed +`cocolic` executable (use its full path if it is not on `PATH`): + +```sh +cocolic --setup-qls +cocolic --check +``` + +Then configure Qt Creator: + +1. Enable **Coco** in the plugin manager (**Help > About Plugins** on Linux) and + restart Qt Creator if requested. The Debian Qt Creator 16.0.1 package used for + this project's coverage check already includes `libCoco.so`. +2. In **Preferences > Coco**, set the Coco installation directory. +3. Clone **Debug** as **DebugCoco**, with its own build directory. Use an ordinary + Debug configuration as the starting point, without GCC `--coverage` flags. +4. In **Projects > Project Settings > Coco Code Coverage**, enable coverage for + this configuration and save/reconfigure. Depending on the Qt Creator version, + the **Coco code coverage** build step also offers **Enable Coverage**. +5. Exclude test sources and Qt/STL headers as appropriate for a library-only + report. Enable `CONFIG+=dbc_qt` to build the optional model tests. +6. Build using Coco's compiler wrappers, then run both test executables. Use the + Coco plugin's report view or CoverageBrowser to inspect the collected results. + +For qmake, the plugin generates `cocoplugin.prf` in the source directory. Coco +must instrument the library subprojects as well as the test executables. +The GCC/LCOV commands above do not collect Coco data. + +See the official [Coco installation and license setup](https://doc.qt.io/coco/installation.html), +[Coco license requirements](https://doc.qt.io/coco/coco-license-utilization.html), +and [Qt Creator Coco configuration](https://doc.qt.io/qtcreator/creator-coco.html). diff --git a/tests/README.md b/tests/README.md index ec060ab..e02ddb8 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,5 +1,8 @@ # QtTest regression tests +For instrumented builds, LCOV HTML reports, Qt Creator coverage configuration, +and Coco installation, see [Measuring code coverage](../README.md#measuring-code-coverage). + Open the repository's root `dbc.pro` in Qt Creator. It contains `core` (a static C++17 library) and `tests` (the `dbc_tests` QtTest executable). Core is built first; tests link its archive rather than compiling its implementation again. diff --git a/tests/dbc_coverage_test.cpp b/tests/dbc_coverage_test.cpp index 2dcb689..2aebc53 100644 --- a/tests/dbc_coverage_test.cpp +++ b/tests/dbc_coverage_test.cpp @@ -3,6 +3,7 @@ #include "dbc_decode_builder.h" #include "dbc_decoder.h" #include "dbc_tree_builder.h" +#include "dbc_statement_handler.h" #include "typefactory.h" #include #include @@ -75,6 +76,11 @@ private Q_SLOTS: void pgn(); void comments_data(); void comments(); + void ignoredCommentStatements_data(); + void ignoredCommentStatements(); + void ambiguousValueReferences_data(); + void ambiguousValueReferences(); + void valueReferenceScope(); void fileFormatting_data(); void fileFormatting(); void missingFile(); @@ -300,6 +306,124 @@ void CoverageTest::comments() { QCOMPARE(parsed.frames[0].signals[0].comment, signalComment.toStdString()); } +void CoverageTest::ignoredCommentStatements_data() { + QTest::addColumn("statement"); + QTest::newRow("empty") << QString(); + QTest::newRow("wrong-keyword") << QStringLiteral("OTHER_ BO_ 1 \"Ignored\";"); + QTest::newRow("keyword-prefix") << QStringLiteral("CM_OTHER BO_ 1 \"Ignored\";"); + QTest::newRow("frame-without-cm") << QStringLiteral("BO_ 1 \"Ignored\";"); + QTest::newRow("missing-subtype") << QStringLiteral("CM_"); + QTest::newRow("global-comment") << QStringLiteral("CM_ \"Ignored\";"); + QTest::newRow("node-comment") << QStringLiteral("CM_ BU_ ECU \"Ignored\";"); + QTest::newRow("environment-comment") << QStringLiteral("CM_ EV_ State \"Ignored\";"); + QTest::newRow("unknown-subtype") << QStringLiteral("CM_ OTHER_ 1 \"Ignored\";"); +} + +void CoverageTest::ignoredCommentStatements() { + QFETCH(QString, statement); + DbcDatabase database; + FrameInfo frame; + frame.canId = 1U; + frame.name = "Frame"; + frame.comment = "Existing frame comment"; + SignalInfo signal; + signal.name = "State"; + signal.comment = "Existing signal comment"; + frame.signals.push_back(signal); + database.frames.push_back(frame); + DbcParseContext context{database, &database.frames[0]}; + + // The handler is public: callers may bypass keyword dispatch. Ignored input + // must preserve existing metadata, including comments already assigned. + DbcCommentHandler().Handle(statement.toStdString(), context); + QCOMPARE(database.frames.size(), std::size_t(1)); + QCOMPARE(database.frames[0].canId, 1U); + QCOMPARE(database.frames[0].name, frame.name); + QCOMPARE(database.frames[0].comment, frame.comment); + QCOMPARE(database.frames[0].signals.size(), std::size_t(1)); + QCOMPARE(database.frames[0].signals[0].name, signal.name); + QCOMPARE(database.frames[0].signals[0].comment, signal.comment); + QCOMPARE(context.currentFrame, &database.frames[0]); +} + +void CoverageTest::ambiguousValueReferences_data() { + QTest::addColumn("duplicateFrames"); + QTest::addColumn("extended"); + QTest::newRow("duplicate-standard-frames") << true << false; + QTest::newRow("duplicate-extended-frames") << true << true; + QTest::newRow("duplicate-signals-in-standard-frame") << false << false; + QTest::newRow("duplicate-signals-in-extended-frame") << false << true; +} + +void CoverageTest::ambiguousValueReferences() { + QFETCH(bool, duplicateFrames); + QFETCH(bool, extended); + DbcDatabase database; + FrameInfo frame; + frame.canId = 1U; + frame.name = "Frame"; + frame.isExtended = extended; + SignalInfo signal; + signal.name = "State"; + frame.signals.push_back(signal); + if (!duplicateFrames) frame.signals.push_back(signal); + database.frames.push_back(frame); + if (duplicateFrames) database.frames.push_back(frame); + + // ParseFile rejects duplicate definitions earlier. Construct the database + // through the public metadata API to exercise the resolver's own checks. + DbcParseContext context{database, nullptr}; + context.lineNumber = 37U; + const std::string rawId = extended ? "2147483649" : "1"; + DbcValueDescriptionHandler().Handle("VAL_ " + rawId + " State 0 \"Off\";", context); + QCOMPARE(context.pendingValues.size(), std::size_t(1)); + QCOMPARE(context.pendingValues[0].lineNumber, std::size_t(37)); + const std::string path = "manual database.dbc"; + try { + context.ResolveValueDescriptions(path); + QFAIL("The resolver must reject an ambiguous target"); + } catch (const std::runtime_error &error) { + const std::string message = duplicateFrames ? "Ambiguous VAL_ frame reference." + : "Ambiguous VAL_ signal reference: State"; + QCOMPARE(std::string(error.what()), path + ":37: " + message); + } + // Ambiguity must be detected before applying any descriptions. + for (const auto &candidate : database.frames) + for (const auto &candidateSignal : candidate.signals) + QVERIFY(candidateSignal.valueDescriptions.empty()); +} + +void CoverageTest::valueReferenceScope() { + DbcDatabase database; + FrameInfo frame; + frame.canId = 1U; + SignalInfo signal; + signal.name = "State"; + frame.signals.push_back(signal); + database.frames.push_back(frame); + frame.isExtended = true; + database.frames.push_back(frame); + frame.canId = 2U; + frame.isExtended = false; + database.frames.push_back(frame); + + DbcParseContext context{database, nullptr}; + const DbcValueDescriptionHandler handler; + context.lineNumber = 3U; + handler.Handle("VAL_ 1 State 0 \"Standard\";", context); + context.lineNumber = 7U; + handler.Handle("VAL_ 2147483649 State 0 \"Extended\";", context); + context.ResolveValueDescriptions("manual database.dbc"); + + const auto *standard = FindValueDescription(database.frames[0].signals[0].valueDescriptions, std::uint64_t(0)); + const auto *extended = FindValueDescription(database.frames[1].signals[0].valueDescriptions, std::uint64_t(0)); + QVERIFY(standard != nullptr); + QVERIFY(extended != nullptr); + QCOMPARE(*standard, std::string("Standard")); + QCOMPARE(*extended, std::string("Extended")); + QVERIFY(database.frames[2].signals[0].valueDescriptions.empty()); +} + void CoverageTest::fileFormatting_data() { QTest::addColumn("text"); QTest::addColumn("frames"); QTest::newRow("empty") << QString::fromUtf8("") << 0U;