Merge branch 'development' into 'master'

Merge old repository

I had to overwrite the code of the new repo manually, so I added this merge request to check whether I have overridden something we had already changed in the new repository.

To-do:
- [x] Handling of parallel HDF5
- [x] Add wrapper for calling parallel RFG with mpirun

See merge request !2

.gitignore

 # Exclude build folder
 build-cmake/
 
-# Exclude the parsed mini file
-testing/convergence.mini
+# Exclude generated files
+testing/*.mini
+python/parfield/wrapper/pf_from_file.py
+python/testtools/wrapper/test_dorie.py
+python/testtools/wrapper/test_dorie_pfg.py
 
 # Ignore temporary and auto-generated files #
 *~

.gitlab-ci.yml

@@ -12,7 +12,7 @@ stages:
 main_job:
   script:
     - ./dune-common/bin/dunecontrol --only=dorie all
-    - ./dune-common/bin/dunecontrol --only=dorie make install
+    - export PATH=/opt/dune/dorie/build-cmake/bin:$PATH
     - ARGS="--output-on-failure" ./dune-common/bin/dunecontrol --only=dorie make test
   stage: main
 
@@ -21,6 +21,6 @@ update_dune_job:
     - ./dune-common/bin/dunecontrol update || true
     - ./dune-common/bin/dunecontrol exec "rm -rf build-cmake"
     - ./dune-common/bin/dunecontrol all
-    - ./dune-common/bin/dunecontrol make install
+    - export PATH=/opt/dune/dorie/build-cmake/bin:$PATH
     - ARGS="--output-on-failure" ./dune-common/bin/dunecontrol --only=dorie make test
   stage: main

CMakeLists.txt

@@ -33,7 +33,6 @@ dune_enable_all_packages()
 dune_require_cxx_standard(MODULE "dorie" VERSION 14)
 
 # add subdirectories
-add_subdirectory("src")
 add_subdirectory("bin")
 add_subdirectory("m4")
 add_subdirectory("dune")

README.md

@@ -76,7 +76,7 @@ Depending on your system configuration, there will be more packages necessary to
 | [dune-python](https://gitlab.dune-project.org/quality/dune-python) | releases/2.4
 | [dune-testtools](https://gitlab.dune-project.org/quality/dune-testtools) | releases/2.4
 
-If you also want to build the documentation, you will additionally need to install Doxygen.
+If you also want to build the documentation, you will additionally need to install Doxygen, Sphinx and Breathe.
 
 ### Manual Installation on Ubuntu
 The following instructions follow the basic installation on a fresh Ubuntu system and are similarly executed to create the DORiE Docker image.
@@ -84,14 +84,15 @@ The following instructions follow the basic installation on a fresh Ubuntu syste
 1. Install third party packages:
     
         apt update 
-        apt install autoconf bison cmake doxygen flex gcc-5 g++-5 gfortran-5 \
+        apt install autoconf bison cmake flex gcc-5 g++-5 gfortran-5 \
                     git libatlas-base-dev libfftw3-dev libfftw3-mpi-dev \
                     libfreetype6-dev libhdf5-mpi-dev libopenmpi-dev libpng-dev \
-                    libsuperlu-dev libxft-dev python3-dev python3-pip
+                    libsuperlu-dev libxft-dev python-dev python-pip python-sphinx \
+                    python-breathe doxygen
 
 2. Install the necessary Python packages using `pip`:
         
-        python3 -m pip install virtualenv
+        python -m pip install virtualenv
 
 3. Clone the specified version of UG into a suitable folder on your machine. Install it by calling
 

bin/CMakeLists.txt

-configure_file(dorie.in dorie @ONLY)
-install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/dorie DESTINATION ${CMAKE_INSTALL_BINDIR})
+configure_file(dorie.in ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/dorie @ONLY)
+# copy the temporary file into the final destination, setting the permissions
+file(COPY ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/dorie
+  DESTINATION ${CMAKE_CURRENT_BINARY_DIR}
+  FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ
+  GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/dorie DESTINATION ${CMAKE_INSTALL_BINDIR} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ
+  GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)

bin/dorie.in

-#!/bin/bash
-
-###### set by CMake ########
-DORIEDIR=@CMAKE_BINARY_DIR@
-MPIRUN=@MPIEXEC@
-############################
-
-DORIE_EXECUTABLE=$DORIEDIR/src/dorie
-PARAMETERDIR=$DORIEDIR/doc/parameters
-DORIE_PYTHON=$DORIEDIR/dune-env
-
-run_dorie_sequential()
-{
-	echo "###################################################"
-	echo "EXECUTING DORIE IN SEQUENTIAL MODE (Single Process)"
-	echo "###################################################"
-	$DORIE_EXECUTABLE $1
-}
-
-run_dorie_parallel()
-{
-	echo "################################"
-	echo "EXECUTING DORIE IN PARALLEL MODE"
-	echo "################################"
-	$MPIRUN -n $2 $DORIE_EXECUTABLE $1
-}
-
-initialize_folder()
-{
-	echo "###################"
-	echo "INITIALIZING FOLDER"
-	echo "###################"
-	cp -iv $PARAMETERDIR/default.ini parameter.ini
-	cp -iv $PARAMETERDIR/default_bc.dat bc.dat
-	echo "Initialization done"
-}
-
-plot_vtk()
-{
-	$DORIE_PYTHON plot_vtk.py --vtk $1 --var $2
-}
-
-###################
-
-# CHECK IF DORIE WAS INSTALLED
-if [[ ! -e $DORIE_EXECUTABLE ]]; then
-	echo -e "Error: DORiE executable not found at $DORIE_EXECUTABLE.\nMake sure you have built and installed DORiE."
-  exit
-fi
-
-RUN=0
-PLOT=0
-CREATE=0
-PARALLEL=0
-
-# NO ARGUMENTS SUPPLIED
-if [[ $# = 0 ]]; then
-	echo "Usage: dorie COMMAND [COMMAND-OPTIONS]. For more information try 'dorie help'"
-	exit
-fi
-
-# PARSE EACH ARGUMENT
-
-case $1 in
-
-	run)
-		RUN=1
-		shift
-		if [[ $# > 0 ]]; then
-			INIFILE=$1
-		else
-			echo "Error: No parameter file specified"
-			exit
-		fi
-		shift
-		if [[ $# > 0 ]]; then
-			case $1 in
-
-				--parallel|--par)
-					shift
-					if [[ $# > 0 ]]; then
-						PARALLEL=$1
-					else
-						echo "Error: Specify the number of threads when using --parallel"
-						exit
-					fi
-				;;
-
-				--sequential|--seq)
-					PARALLEL=0
-				;;
-
-				*)
-					echo "Error: Unrecognized option $1"
-					exit
-				;;
-
-			esac
-		fi
-	;;
-
-	plot)
-	  PLOT=1
-		shift
-		if [[ $# = 0 ]]; then
-			echo "Error: Specify a VTK file to plot"
-			exit
-		fi
-
-		while [[ $# > 0 && ! $1 = "--var" ]]; do
-			if [[ -e $1 && -f $1 && -s $1 ]]; then
-				VTKFILE+="$1 "
-			else
-				echo "Error: Invalid VTK file $1"
-				exit
-			fi
-			shift
-		done
-
-		if [[ $1 = "--var" ]]; then
-			shift
-			VARIABLES=""
-			while [[ $# > 0 ]]; do
-				VARIABLES+="$1 "
-				shift
-			done
-		fi
-	;;
-
-	create)
-		CREATE=1
-	;;
-
-	help)
-		echo -e "Usage: dorie COMMAND [COMMAND-OPTIONS]\n"
-		echo -e "COMMANDS:"
-		echo -e "  'run INIFILE [OPTIONS]'  starts DORiE with parameter file INIFILE"
-		echo -e "  OPTIONS:"
-		echo -e "    '--parallel NPROC'  starts DORiE in parallel with NPROC processes"
-		echo -e "    '--sequential'      starts DORiE in sequential mode (default)"
-		echo -e "  'create'                 creates dummy parameter files in the current directory"
-		echo -e "  'plot VTK [--var VAR]'   plots the variables VAR (default: all variables) from the VTK file(s) VTK"
-		echo -e "  'help'                   displays this message"
-
-		exit
-	;;
-
-	*)
-		echo -e "Error: Command not recognized: $1. For more information run 'dorie help'"
-		exit
-	;;
-esac
-
-# RUN DORIE
-if [[ $RUN = 1 ]]; then
-	if [[ -e $INIFILE && -f $INIFILE && -s $INIFILE ]]; then
-		if [[ $PARALLEL > 0 ]]; then
-			run_dorie_parallel $INIFILE $PARALLEL
-		else
-			run_dorie_sequential $INIFILE
-		fi
-	else
-		echo "Error: Invalid parameter file $INIFILE"
-	fi
-	exit
-fi
-
-# INITIALIZE FOLDER
-if [[ $CREATE = 1 ]]; then
-	initialize_folder
-	exit
-fi
-
-# PLOT VTK FILES
-if [[ $PLOT = 1 ]]; then
-	plot_vtk "$VTKFILE" "$VARIABLES"
-	exit
-fi
-
-
+#!/usr/bin/env python
+
+import os
+import sys
+import shutil
+import argparse
+import warnings
+import subprocess
+import multiprocessing
+
+try: # Python 2.x compatibility
+    input = raw_input
+except NameError:
+    pass
+
+# paths set by cmake
+DORIEDIR = "@CMAKE_BINARY_DIR@"
+MPIEXEC = "@MPIEXEC@"
+MPIEXEC_NUMPROC_FLAG = "@MPIEXEC_NUMPROC_FLAG@"
+MPIEXEC_PREFLAG = "@MPIEXEC_PREFLAG@"
+MPIEXEC_POSTFLAGS = "@MPIEXEC_POSTFLAGS@"
+#
+
+# set some paths
+DORIE_EXECUTABLE = os.path.join(DORIEDIR, "dune/dorie/dorie")
+PARAMETERDIR = os.path.join(DORIEDIR, "doc/default_files")
+DORIE_PYTHON = os.path.join(DORIEDIR, "dune-env")
+MPIRUN = lambda nproc, exe, *args: [k for k in [MPIEXEC,MPIEXEC_NUMPROC_FLAG,str(nproc),MPIEXEC_PREFLAG,str(exe),MPIEXEC_POSTFLAGS] + list(args) if k]
+
+def run(args):
+    if not os.path.isfile(args["config"]):
+        raise IOError("Configuration file {} not found".format(args["config"]))
+    try:
+        if args["parallel"] == 1:
+            subprocess.check_call([DORIE_EXECUTABLE, args["config"]])
+        else:
+            subprocess.check_call(MPIRUN(args["parallel"],DORIE_EXECUTABLE,args["config"]))
+    except subprocess.CalledProcessError:
+        print("Error while running DORiE")
+        sys.exit(1)
+
+def create(args):
+    for f in ("config.ini", "parfield.ini", "2d_infiltr.bcdat", "3d_infiltr.bcdat"):
+        newfile = os.path.join(os.getcwd(),f)
+        if os.path.exists(newfile) and not args["force"]:
+            override = (input("File {} exists. Override? [Y/n] ".format(f)) or "Y") in ["Y","y"]
+            if not override:
+                continue
+        shutil.copyfile(os.path.join(PARAMETERDIR, f), newfile)
+    print("Folder initialized")
+
+def pfg(args):
+    if not os.path.isfile(args["config"]):
+        raise IOError("Configuration file {} not found".format(args["config"]))
+    if args["parallel"] == 1:
+        subprocess.call([DORIE_PYTHON, "pf_from_file.py", args["config"]])
+    else:
+        subprocess.call([DORIE_PYTHON, "pf_from_file.py", args["config"], "--parallel", str(args["parallel"])])
+
+def plot(args):
+    if not os.path.isfile(args["vtk"]):
+        raise IOError("File {} not found".format(args["vtk"]))
+    if args["var"]:
+        subprocess.call([DORIE_PYTHON, "plot_vtk.py", "-f", args["vtk"], "--var", args["var"]])
+    else:
+        subprocess.call([DORIE_PYTHON, "plot_vtk.py", "-f", args["vtk"]])
+
+if __name__ == "__main__": # parse command line and call command handler
+    try:
+        with warnings.catch_warnings(record=True) as warn:
+            parser = argparse.ArgumentParser(description="DORiE command line interface",
+                                             epilog="For more information check our online documentation: "
+                                                    "http://dorie-docs.bitballoon.com (password: 'richards')."
+                                            )
+            subparsers = parser.add_subparsers(title="Commands",dest="command")
+            subparsers.required = True
+
+            parser_create = subparsers.add_parser('create',
+                                                  description="Copy example configuration to current folder.",
+                                                  help="Copy example configuration to current folder.")
+            parser_create.add_argument('-f','--force', help="Override existing files.", action="store_true")
+            parser_create.set_defaults(func=create)
+
+            parser_pfg = subparsers.add_parser('pfg', help="Start parameter field generator.",
+                                               description="Start parameter field generator.",
+                                               usage="%(prog)s <config> [-h] [-p [N]]")
+            parser_pfg.add_argument('config',
+                                    help="Configuration file for the parameter field generator. "
+                                         "Can be created with 'dorie create'.")
+            parser_pfg.add_argument('-p','--parallel', metavar='N', nargs='?', default=1,
+                                    const=multiprocessing.cpu_count(), type=int, required=False,
+                                    help="Run in parallel on N processes. "
+                                         "If N is not specified, run on all available CPU threads.")
+            parser_pfg.set_defaults(func=pfg)
+
+            parser_run = subparsers.add_parser('run', help="Run DORiE.", description="Run DORiE.",
+                                               usage="%(prog)s <config> [-h] [-p [N]]")
+            parser_run.add_argument('config',help="DORiE configuration file. Can be created with 'dorie create'.")
+            parser_run.add_argument('-p','--parallel', metavar='N', nargs='?', default=1,
+                                    const=multiprocessing.cpu_count(), type=int, required=False,
+                                    help="Run in parallel on N processes. "
+                                         "If N is not specified, run on all available CPU threads.")
+            parser_run.set_defaults(func=run)
+
+            parser_plot = subparsers.add_parser('plot', help="Plot a preview of a VTK file created by DORiE.",
+                                                description="Plot a preview of a VTK file created by DORiE.",
+                                                usage="%(prog)s <vtk> [-h] [--v [VAR [VAR ...]]]")
+            parser_plot.add_argument('vtk', help="Input VTK file.")
+            parser_plot.add_argument('-v','--var', nargs='*', required=False,
+                                     help="Plot only given variables. If not specified, all variables are plotted.")
+            parser_plot.set_defaults(func=plot)
+
+            args = parser.parse_args()
+            args.func(vars(args))
+    except Exception as e:
+        print("dorie failed with ({0}) warning(s) and ({1}) error(s)".format(len(warn),1))
+        raise

cmake/modules/CTestTestfile.cmake

-# CMake generated Testfile for 
-# Source directory: /home/dion/codes/dorie/dorie/cmake/modules
-# Build directory: /home/dion/codes/dorie/dorie/cmake/modules
-# 
-# This file includes the relevant testing commands required for 
-# testing this directory and lists subdirectories to be tested as well.

cmake/modules/DorieMacros.cmake

 # File for module specific CMake tests.
 
 # find all required packages
-FIND_PACKAGE(HDF5 REQUIRED COMPONENTS C)
-FIND_PACKAGE(FFTW REQUIRED)
-FIND_PACKAGE(SuperLU REQUIRED)
-FIND_PACKAGE(MPI REQUIRED)
-FIND_PACKAGE(UG REQUIRED)
+FIND_PACKAGE (HDF5 REQUIRED COMPONENTS C)
+if(HDF5_IS_PARALLEL)
+	message(STATUS "Parallel HDF5 library found")
+	add_definitions(-DHDF5_PARALLEL)
+endif()
+FIND_PACKAGE (FFTW REQUIRED)
+FIND_PACKAGE (SuperLU REQUIRED)
+FIND_PACKAGE (MPI REQUIRED)
+FIND_PACKAGE (UG REQUIRED)
 
-FIND_PACKAGE(METIS)
-FIND_PACKAGE(ParMETIS)
+FIND_PACKAGE (METIS)
+FIND_PACKAGE (ParMETIS)
 
 include_directories(${FFTW_INCLUDES})
 include_directories(${HDF5_INCLUDE_DIRS})

cmake/modules/Makefile

-# CMAKE generated file: DO NOT EDIT!
-# Generated by "Unix Makefiles" Generator, CMake Version 3.5
-
-# Default target executed when no arguments are given to make.
-default_target: all
-
-.PHONY : default_target
-
-# Allow only one "make -f Makefile2" at a time, but pass parallelism.
-.NOTPARALLEL:
-
-
-#=============================================================================
-# Special targets provided by cmake.
-
-# Disable implicit rules so canonical targets will work.
-.SUFFIXES:
-
-
-# Remove some rules from gmake that .SUFFIXES does not remove.
-SUFFIXES =
-
-.SUFFIXES: .hpux_make_needs_suffix_list
-
-
-# Suppress display of executed commands.
-$(VERBOSE).SILENT:
-
-
-# A target that is always out of date.
-cmake_force:
-
-.PHONY : cmake_force
-
-#=============================================================================
-# Set environment variables for the build.
-
-# The shell in which to execute make rules.
-SHELL = /bin/sh
-
-# The CMake executable.
-CMAKE_COMMAND = /usr/bin/cmake
-
-# The command to remove a file.
-RM = /usr/bin/cmake -E remove -f
-
-# Escaping for special characters.
-EQUALS = =
-
-# The top-level source directory on which CMake was run.
-CMAKE_SOURCE_DIR = /home/dion/codes/dorie/dorie
-
-# The top-level build directory on which CMake was run.
-CMAKE_BINARY_DIR = /home/dion/codes/dorie/dorie
-
-#=============================================================================
-# Targets provided globally by CMake.
-
-# Special rule for the target test
-test:
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running tests..."
-	/usr/bin/ctest --force-new-ctest-process $(ARGS)
-.PHONY : test
-
-# Special rule for the target test
-test/fast: test
-
-.PHONY : test/fast
-
-# Special rule for the target package
-package: preinstall
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Run CPack packaging tool..."
-	cd /home/dion/codes/dorie/dorie && /usr/bin/cpack --config ./CPackConfig.cmake
-.PHONY : package
-
-# Special rule for the target package
-package/fast: package
-
-.PHONY : package/fast
-
-# Special rule for the target edit_cache
-edit_cache:
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake cache editor..."
-	/usr/bin/cmake-gui -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
-.PHONY : edit_cache
-
-# Special rule for the target edit_cache
-edit_cache/fast: edit_cache
-
-.PHONY : edit_cache/fast
-
-# Special rule for the target package_source
-package_source:
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Run CPack packaging tool for source..."
-	cd /home/dion/codes/dorie/dorie && /usr/bin/cpack --config ./CPackSourceConfig.cmake /home/dion/codes/dorie/dorie/CPackSourceConfig.cmake
-.PHONY : package_source
-
-# Special rule for the target package_source
-package_source/fast: package_source
-
-.PHONY : package_source/fast
-
-# Special rule for the target rebuild_cache
-rebuild_cache:
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Running CMake to regenerate build system..."
-	/usr/bin/cmake -H$(CMAKE_SOURCE_DIR) -B$(CMAKE_BINARY_DIR)
-.PHONY : rebuild_cache
-
-# Special rule for the target rebuild_cache
-rebuild_cache/fast: rebuild_cache
-
-.PHONY : rebuild_cache/fast
-
-# Special rule for the target install/strip
-install/strip: preinstall
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing the project stripped..."
-	/usr/bin/cmake -DCMAKE_INSTALL_DO_STRIP=1 -P cmake_install.cmake
-.PHONY : install/strip
-
-# Special rule for the target install/strip
-install/strip/fast: install/strip
-
-.PHONY : install/strip/fast
-
-# Special rule for the target install/local
-install/local: preinstall
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Installing only the local directory..."
-	/usr/bin/cmake -DCMAKE_INSTALL_LOCAL_ONLY=1 -P cmake_install.cmake
-.PHONY : install/local
-
-# Special rule for the target install/local
-install/local/fast: install/local
-
-.PHONY : install/local/fast
-
-# Special rule for the target list_install_components
-list_install_components:
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Available install components are: \"Unspecified\""
-.PHONY : list_install_components
-
-# Special rule for the target list_install_components
-list_install_components/fast: list_install_components
-
-.PHONY : list_install_components/fast
-
-# Special rule for the target install
-install: preinstall
-	@$(CMAKE_COMMAND) -E cmake_echo_color --switch=$(COLOR) --cyan "Install the project..."
-	/usr/bin/cmake -P cmake_install.cmake
-.PHONY : install
-
-# Special rule for the target install