Commit d9a9b8a4 authored by Lukas Riedel's avatar Lukas Riedel

Merge branch '99-build-unit-tests-in-separate-pipeline-job' into 'master'

Resolve "Build unit tests in separate pipeline job"

Closes #93 and #99

See merge request !98
parents cd157b14 df1f8410
......@@ -2,7 +2,6 @@
build-cmake/
# Exclude generated files
test/*.mini
python/parfield/wrapper/pf_from_file.py
python/testtools/wrapper/test_dorie.py
python/testtools/wrapper/test_dorie_pfg.py
......
......@@ -81,12 +81,12 @@ prep:update-dune-clang:
# --- Build jobs ---
build:main:
build:system-tests: &build-tests
stage: build
script:
- CMAKE_FLAGS="$CMAKE_FLAGS"
$DUNECONTROL --only=dorie all
- $DUNECONTROL --only=dorie make build_tests
$DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make build_system_tests
- $DUNECONTROL --only=dorie make doc
artifacts:
name: "$CI_JOB_NAME"
......@@ -94,6 +94,14 @@ build:main:
- $CI_PROJECT_DIR/build-cmake
expire_in: 1 day
build:unit-tests:
<<: *build-tests
script:
- CMAKE_FLAGS="$CMAKE_FLAGS
-DCMAKE_BUILD_TYPE=Debug"
$DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make build_unit_tests
build:debug: &debug
stage: build
allow_failure: true
......@@ -103,7 +111,7 @@ build:debug: &debug
-DCMAKE_BUILD_TYPE=Debug
-DCMAKE_CXX_FLAGS_DEBUG='-Werror'"
$DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make dorie_mass_conservation
- $DUNECONTROL --only=dorie make build_unit_tests
build:debug-clang:
<<: *debug
......@@ -116,14 +124,14 @@ build:debug-clang:
-DCMAKE_CXX_COMPILER=clang++
-DCMAKE_CXX_FLAGS_DEBUG='-Werror'"
$DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make dorie_mass_conservation
- $DUNECONTROL --only=dorie make build_unit_tests
# --- Tests ---
test:exec_cmds: &test
test:st:exec_cmds: &test
stage: test
dependencies:
- build:main
- build:system-tests
script:
- $DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make test_dorie_exec
......@@ -133,13 +141,13 @@ test:exec_cmds: &test
- $CI_PROJECT_DIR/build-cmake/test
expire_in: 1 day
test:ode:
test:st:ode:
<<: *test
script:
- $DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make test_run_ode
test:parallel:
test:st:parallel:
<<: *test
tags:
- multicore
......@@ -147,24 +155,39 @@ test:parallel:
- $DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make test_run_parallel
test:reference:
test:st:reference:
<<: *test
script:
- $DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make test_run_ref
test:mass_conserve:
test:st:mass_conserve:
<<: *test
script:
- $DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make test_mass_conservation
test:new-parameters:
test:unit-tests:
<<: *test
dependencies:
- build:unit-tests
script:
# install coverage reporter
- $DUNECONTROL --only=dorie bexec
"./run-in-dune-env python3 -m pip install gcovr"
- $DUNECONTROL --only=dorie configure
- $DUNECONTROL --only=dorie make test_param
- $DUNECONTROL --only=dorie make unit_tests
# report coverage
- cd $CI_PROJECT_DIR/build-cmake
- mkdir -p test/coverage
- ./run-in-dune-env gcovr
--root ../
--html --html-details
-o test/coverage/coverage.html
# run again for coverage report in GitLab
- ./run-in-dune-env gcovr
--root ../
coverage: '/^TOTAL.*\s+(\d+\%)$/'
# --- Deploy jobs ---
deploy:dockerhub-devel: &deploy
......@@ -202,7 +225,7 @@ deploy:sphinx-docs:
only:
- tags@dorie/dorie
dependencies:
- build:main
- build:system-tests
before_script:
# install the netfly CLI
- apt-get install -y golang-go golang-glide
......
......@@ -9,6 +9,13 @@
* New classes representing parameterizations. Every parameterization must now
derive from `RichardsParameterization`, and return function objects
representing parameterization functions.
* CMake module `DorieTesting.cmake`. The module contains multiple functions
which facilitate registering tests in the two new testing categories:
System tests and Unit tests. Unit tests are compiled with appropriate flags
for code coverage reports.
* Code coverage reports for unit tests. Detailed reports can be retrieved from
the artifacts of the `test:unit-tests` test in the folder
`build-cmake/test/coverage`. The total coverage is reported to GitLab.
### Changed
* `Simulation` is renamed `RichardsSimulation` and moved to
......@@ -41,6 +48,9 @@
usual `VTKSequenceWriter` for grid functions called
`GridFunctionVTKSequenceWriter`.
* Swich to the stable release branch `releases/2.6` of `dune-testtools`.
* The `build` CI stage now explicitly builds system tests and unit tests. The
job `build:main` has been removed. The `build:debug` jobs now build the
unit test targets.
### Deprecated
* The configuration file key `[parameters.interpolation]` is deprecated due to
......
......@@ -247,18 +247,24 @@ A debugger needs special security privileges usually not provided by the Docker
The debugger `gdb` is pre-installed in the Docker container.
### Running System Tests
DORiE includes system tests for comparing its results the ones of ODE solvers or
former versions of itself. They ensure that DORiE is running correctly and
producing the expected results. If you installed DORiE, you have to make sure
that all test executables are built by calling
dunecontrol --only=dorie make build_tests
You then execute the system tests with
ARGS="--output-on-failure" dunecontrol --only=dorie make test
You will be informed whether each test has been passed or failed, and you may find additional output in the DORiE build directory.
DORiE includes a testing system for comparing its results the ones of ODE solvers
or former versions of itself. This ensures that DORiE is running correctly and
producing the expected results. We distinguish _unit tests_ for testing certain
features of the code, and _system tests_ for verifying the results of the
final application. As system tests require executing the DUNE solvers,
it is recommended to build them in a `Release` environment.
| Test category | Build tests | Execute tests | Recommended build type |
| ----- | ----- | ----- |
| Unit tests | `make build_unit_tests` | `make_unit_tests` | `Debug` |
| System tests | `make build_system_tests` | `make_system_tests` | `Release` |
The `make` commands are to be executed from within the `build-cmake` directory.
The unit tests automatically include compiler flags for code coverage reports.
After building them with `Debug` flags and executing them, you can
retrieve code coverage information using the
[`gcovr`](https://gcovr.com/index.html) utility.
### Further Help
[Open an issue](https://ts-gitlab.iup.uni-heidelberg.de/dorie/dorie/issues/new), or write to the [DORiE developer mailing list](mailto:dorieteam@iup.uni-heidelberg.de).
# File for module specific CMake tests.
# find all required packages
FIND_PACKAGE (HDF5 REQUIRED)
if(NOT HDF5_IS_PARALLEL)
......@@ -24,3 +22,6 @@ list (APPEND DUNE_LIBS
${HDF5_LIBRARIES}
${YAML_CPP_LIBRARIES})
MESSAGE(STATUS "DUNE Libraries: ${DUNE_LIBS}")
# Add DORiE testing functions
include(DorieTesting)
# Add the unit test build target and test command
add_custom_target(build_unit_tests)
add_custom_target(unit_tests
COMMAND ctest --output-on-failure --tests-regex ^ut.+$
)
# Add the system test build target and test command
add_custom_target(build_system_tests)
add_custom_target(system_tests
COMMAND ctest --output-on-failure --exclude-regex ^ut.+$
)
#
# .. cmake_function:: add_coverage_links
#
# This function adds the appropriate compiler and linker flags for creating
# a coverage report from the resulting object files of the specified targets.
# Signature:
# `add_coverage_links(<target>...)`
#
function(add_coverage_links)
foreach(target ${ARGV})
target_compile_options(${target} PRIVATE --coverage)
target_link_libraries(${target} PRIVATE --coverage)
endforeach()
endfunction()
#
# .. cmake_function:: dorie_add_unit_test
#
# .. cmake_param:: NAME
# :single:
# :required:
#
# The name of the resulting test executable and target.
#
# .. cmake_param:: TARGET
# :single:
#
# The target this test applies to. This is only required if no SOURCES
# are specified.
#
# This function serves as wrapper around the function `dune_add_test` which
# registers test for existing targets or adds new test executables from the
# given source files. This function additionally registers the tests as unit
# tests within DORiE and adds flags for coverage reports. Notice that
# `dune_add_test` requires more parameters than this function alone.
#
# Use this function exactly like `dune_add_test`.
#
function(dorie_add_unit_test)
set(SINGLE NAME TARGET)
cmake_parse_arguments(UNIT_TEST "" "${SINGLE}" "" ${ARGN})
# use name prefix for test
if(NOT UNIT_TEST_NAME)
message(SEND_ERROR "No unit test name specified!")
string(PREPEND UNIT_TEST_NAME ut-)
# forward to dune function
dune_add_test(NAME ${UNIT_TEST_NAME}
${UNIT_TEST_UNPARSED_ARGUMENTS})
# get (build) target name if no target was given
if(NOT UNIT_TEST_TARGET)
set(UNIT_TEST_TARGET ${UNIT_TEST_NAME})
endif()
# add to build target and employ compile options
add_coverage_links(${UNIT_TEST_TARGET})
add_dependencies(build_unit_tests ${UNIT_TEST_TARGET})
endfunction()
#
# .. cmake_function:: dorie_add_metaini_test
#
# .. cmake_param:: UNIT_TEST
# :option:
#
# Registers the created tests as unit tests, including coverage flags.
# If not specified, the tests are registered as system tests.
#
# .. cmake_param:: TARGET
# :single:
#
# The existing target to apply these tests to. This is incompatible to
# the option `UNIT_TEST` and the parameter `BASENAME`, because the base
# name of the tests will automatically be set to the target name.
#
# .. cmake_param:: METAINI
# :single:
# :required:
#
# The meta-ini _input_ file for this test.
#
# .. cmake_param:: SCRIPT
# :single:
#
# The Python script to call for this test. The script has to be installed
# into the CMake `virtualenv`. Defaults to `dune_execute.py` for unit
# tests and `test_dorie.py` for system tests.
#
# .. cmake_param:: BASENAME
# :single:
#
# The basename for tests created from source files. This option is ignored
# (by `dune_add_system_test`) if `TARGET` is specified.
#
# .. cmake_param:: CREATED_TARGETS
# :single:
#
# The variable to hold the created CMake targets for post-processing.
#
# This function serves as wrapper around the function `dune_add_system_test`
# which registers test for existing targets or adds new test executables from
# the given source files by expanding meta ini files.
# This function can register the tests as unit or as system tests within
# DORiE. Notice that `dune_add_system_test` requires more parameters
# than this function alone.
#
# Use this function like `dune_add_system_test`, considering the options
# given above.
#
function(dorie_add_metaini_test)
set(OPTIONS UNIT_TEST)
set(SINGLE TARGET METAINI SCRIPT BASENAME CREATED_TARGETS)
cmake_parse_arguments(SYSTEM_TEST "${OPTIONS}" "${SINGLE}" "" ${ARGN})
if(NOT SYSTEM_TEST_METAINI)
message(SEND_ERROR "No meta ini file given!")
endif()
# configure meta ini file or just copy.
get_filename_component(metaini-name ${SYSTEM_TEST_METAINI} NAME_WE)
get_filename_component(metaini-extension ${SYSTEM_TEST_METAINI} EXT)
if(metaini-extension EQUAL ".mini.in")
configure_file(${SYSTEM_TEST_METAINI} ${metaini-name}.mini)
set(SYSTEM_TEST_METAINI "${metaini-name}.mini")
else()
configure_file(${SYSTEM_TEST_METAINI} ${SYSTEM_TEST_METAINI})
endif()
if(SYSTEM_TEST_TARGET AND SYSTEM_TEST_UNIT_TEST)
message(SEND_ERROR "Specifying TARGET is incompatible to option UNIT_TEST!")
endif()
# Set at least the prefix as basename
if(NOT SYSTEM_TEST_TARGET)
if(SYSTEM_TEST_UNIT_TEST)
string(PREPEND SYSTEM_TEST_BASENAME ut-)
else()
string(PREPEND SYSTEM_TEST_BASENAME st-)
endif()
endif()
# default script for system tests
if(NOT SYSTEM_TEST_SCRIPT)
if(NOT SYSTEM_TEST_UNIT_TEST)
set(SYSTEM_TEST_SCRIPT test_dorie.py)
endif()
endif()
# forward to DUNE function
dune_add_system_test(
${SYSTEM_TEST_UNPARSED_ARGUMENTS}
TARGET ${SYSTEM_TEST_TARGET}
BASENAME ${SYSTEM_TEST_BASENAME}
INIFILE ${CMAKE_CURRENT_BINARY_DIR}/${SYSTEM_TEST_METAINI}
SCRIPT ${SYSTEM_TEST_SCRIPT}
CREATED_TARGETS created_targets
)
# report created targets to parent scope
set(${SYSTEM_TEST_CREATED_TARGETS} ${created_targets} PARENT_SCOPE)
# add dependencies and flags
if(SYSTEM_TEST_UNIT_TEST)
add_coverage_links(${created_targets})
add_dependencies(build_unit_tests ${created_targets})
else()
add_dependencies(build_system_tests ${created_targets})
endif()
endfunction()
......@@ -9,11 +9,12 @@ ARG CXX=g++
WORKDIR /opt/dune
ADD . /opt/dune/dorie/
RUN MAKE_FLAGS="-j ${PROCNUM}" \
CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=${CC} -DCMAKE_CXX_COMPILER=${CXX} -DDUNE_PYTHON_VIRTUALENV_SETUP=True -DDUNE_PYTHON_ALLOW_GET_PIP=True" \
./dune-common/bin/dunecontrol --only=dorie all \
RUN CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=${CC} -DCMAKE_CXX_COMPILER=${CXX} -DDUNE_PYTHON_VIRTUALENV_SETUP=True -DDUNE_PYTHON_ALLOW_GET_PIP=True" \
./dune-common/bin/dunecontrol --only=dorie configure \
&& MAKE_FLAGS="-j ${PROCNUM}" \
./dune-common/bin/dunecontrol --only=dorie make dorie-rfg dorie
&& ./dune-common/bin/dunecontrol --only=dorie exec \
"rm build-cmake/dune/dorie/impl/libdorie-impl.a build-cmake/dune/dorie/impl/CMakeFiles/dorie-impl.dir/*.cc.o"
"rm build-cmake/dune/dorie/impl/libdorie-impl.a build-cmake/dune/dorie/impl/CMakeFiles/dorie-impl.dir/*.cc.o"
ENV PATH="/opt/dune/dorie/build-cmake/bin:${PATH}"
WORKDIR /sim
ENTRYPOINT ["/opt/dune/dorie/build-cmake/bin/dorie"]
\ No newline at end of file
message(STATUS "Handling system tests")
MESSAGE(STATUS "DUNE Libraries: ${DUNE_LIBS}")
function(dorie_add_system_test target metaini)
configure_file(${metaini}.in ${CMAKE_CURRENT_LIST_DIR}/${metaini})
dune_add_system_test(TARGET ${target} INIFILE ${metaini} SCRIPT test_dorie.py)
endfunction()
function(dorie_add_system_test_dependency test1 test2)
set_tests_properties(${test1} PROPERTIES DEPENDS ${test2})
endfunction()
# dorie run: ODE tests
dorie_add_system_test(dorie ode_homogeneous_sand.mini)
dorie_add_system_test(dorie ode_homogeneous_silt.mini)
dorie_add_system_test(dorie ode_layered.mini)
dorie_add_metaini_test(TARGET dorie METAINI ode_homogeneous_sand.mini.in)
dorie_add_metaini_test(TARGET dorie METAINI ode_homogeneous_silt.mini.in)
dorie_add_metaini_test(TARGET dorie METAINI ode_layered.mini.in)
# add target for ODE tests
add_custom_target(test_run_ode
COMMAND ctest --output-on-failure --tests-regex ^dorie_ode.+$)
# dorie run: Reference tests
dorie_add_system_test(dorie-rfg parfield_muphi.mini)
dorie_add_system_test(dorie muphi.mini)
dorie_add_metaini_test(TARGET dorie-rfg METAINI parfield_muphi.mini.in)
dorie_add_metaini_test(TARGET dorie METAINI muphi.mini.in)
set_tests_properties(dorie-rfg_ref_muphi_pfg PROPERTIES FIXTURES_SETUP muphi_ref)
set_tests_properties(dorie_ref_muphi PROPERTIES FIXTURES_REQUIRED muphi_ref)
......@@ -29,14 +19,14 @@ add_custom_target(test_run_ref
COMMAND ctest --output-on-failure --tests-regex ^.+_ref_.+$
)
# dorie_add_system_test(dorie reference_2d.mini)
# dorie_add_system_test(dorie reference_3d.mini)
# dorie_add_system_test(dorie reference_evaporation.mini)
# dorie_add_system_test(dorie reference_interpolators.mini)
# dorie_add_metaini_test(TARGET dorie METAINI reference_2d.mini.in)
# dorie_add_metaini_test(TARGET dorie METAINI reference_3d.mini.in)
# dorie_add_metaini_test(TARGET dorie METAINI reference_evaporation.mini.in)
# dorie_add_metaini_test(TARGET dorie METAINI reference_interpolators.mini.in)
# dorie in parallel
dorie_add_system_test(dorie parallel_reference.mini)
dorie_add_system_test(dorie parallel_reference_compare.mini)
dorie_add_metaini_test(TARGET dorie METAINI parallel_reference.mini.in)
dorie_add_metaini_test(TARGET dorie METAINI parallel_reference_compare.mini.in)
set_tests_properties(dorie_parallel_reference_0000 PROPERTIES FIXTURES_SETUP dorie_par)
set_tests_properties(dorie_parallel_reference_0001 PROPERTIES FIXTURES_SETUP dorie_par)
......@@ -50,50 +40,43 @@ add_custom_target(test_run_parallel
)
# dorie exec tests
dorie_add_system_test(dorie-rfg parfield.mini)
dorie_add_system_test(dorie run.mini)
dorie_add_metaini_test(TARGET dorie-rfg METAINI parfield.mini.in)
dorie_add_metaini_test(TARGET dorie METAINI run.mini.in)
set_tests_properties(dorie-rfg_exec_0000 PROPERTIES FIXTURES_SETUP dorie_run)
set_tests_properties(dorie_exec_run PROPERTIES FIXTURES_REQUIRED dorie_run)
dorie_add_system_test(dorie plot.mini)
dorie_add_system_test_dependency(dorie_exec_plot dorie_exec_run)
dorie_add_metaini_test(TARGET dorie METAINI plot.mini.in)
set_tests_properties(dorie_exec_run PROPERTIES FIXTURES_SETUP dorie_plot)
set_tests_properties(dorie_exec_plot PROPERTIES FIXTURES_REQUIRED dorie_plot)
dorie_add_system_test(dorie create.mini)
dorie_add_metaini_test(TARGET dorie METAINI create.mini.in)
add_custom_target(test_dorie_exec
COMMAND ctest --output-on-failure --tests-regex ^.+_exec.+$
)
# dorie mass conservation
configure_file(mass_conservation.mini.in ${CMAKE_CURRENT_SOURCE_DIR}/mass_conservation.mini)
dune_add_system_test(
dorie_add_metaini_test(
SOURCE test-mass-conservation.cc
BASENAME dorie_mass_conservation
BASENAME mass-conservation
CREATED_TARGETS mc_target
INIFILE mass_conservation.mini
METAINI mass_conservation.mini.in
SCRIPT dune_execute.py
)
message(STATUS "target called ${mc_target}")
dune_target_link_libraries(${mc_target} ${DUNE_LIBS})
add_custom_target(test_mass_conservation
COMMAND ctest --output-on-failure --tests-regex ^dorie_mass_conservation_.+$
COMMAND ctest --output-on-failure --tests-regex ^.+mass-conservation.+$
)
# dorie parameterization test
configure_file(param.mini.in ${CMAKE_CURRENT_SOURCE_DIR}/param.mini)
dune_add_system_test(
dorie_add_metaini_test(UNIT_TEST
SOURCE test-parameterization.cc
BASENAME dorie_parameterization
BASENAME test-parameterization
CREATED_TARGETS par_target
INIFILE param.mini
METAINI param.mini.in
SCRIPT
)
dune_target_link_libraries(${par_target} ${DUNE_LIBS})
add_custom_target(test_param
COMMAND ctest --output-on-failure --tests-regex ^dorie_parameterization_.+$
COMMAND ctest --output-on-failure --tests-regex ^.+test-parameterization_.+$
)
# dune excludes test targets from 'make all'; undo that here where applicable
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment