Add cook book tutorial on random field generator

Explain basics of random field generation and include two main use
cases, Miller-similar media and heterogeneous media. Add input files for theses cases and include excerpts into the docs.

Images are already uploaded to the DORiE repository Wiki.

.gitignore

@@ -5,6 +5,7 @@ build-cmake/
 doc/manual/config-file.rst
 doc/default_files/config.ini
 doc/cookbook/1-infiltration-sand/config.ini
+doc/cookbook/3-random-field/*.ini
 python/dorie/wrapper/pf_from_file.py
 python/dorie/dorie/cli/cmds.py
 test/maps/cell_ids.h5

.gitlab-ci.yml

@@ -233,6 +233,23 @@ test:unit-tests:
       - $CI_PROJECT_DIR/build-cmake/coverage
     expire_in: 1 day
 
+test:examples:
+  stage: test
+  dependencies:
+    - build:system-tests
+  needs: ["build:system-tests"]
+  only:
+    - schedules
+    - master@dorie/dorie
+  script:
+    - $DUNECONTROL --only=dorie configure
+    - $DUNECONTROL --only=dorie make example_tests
+  artifacts:
+    name: "$CI_JOB_NAME"
+    paths:
+      - $CI_PROJECT_DIR/build-cmake/doc
+    expire_in: 1 day
+
 test:python-tests:
   <<: *test
   dependencies: []

CHANGELOG.md

@@ -36,6 +36,7 @@
 * Changes to config file parameters listed per version in user docs !175
 * Control negative transport solution by a check policy !181
 * DG solver for solute transport model !112
+* Cookbook tutorial on using the random field generator !184
 
 ### Changed
 * Data structures for storing and accessing parameter information !55

README.md

@@ -426,9 +426,10 @@ Additionaly, there is a set of tests for the Python module.
 
 | Test category | Build tests | Execute tests | Recommended build type |
 | ------------- | ----------- | ------------- | ---------------------- |
-| Unit tests | `make build_unit_tests` | `make unit_tests` | `Debug`
-| System tests | `make build_system_tests` | `make system_tests` | `Release`
-| Python tests | _Not required_ | `make test_python` | _Any_
+| Unit tests | `make build_unit_tests` | `make unit_tests` | `Debug` |
+| System tests | `make build_system_tests` | `make system_tests` | `Release` |
+| Python tests | _Not required_ | `make test_python` | _Any_ |
+| Cookbook examples (no testing performed) | `make all` | `make example_tests` | `Release` |
 
 The `make` commands are to be executed from within the `build-cmake` directory.
 

cmake/modules/DorieTesting.cmake

@@ -8,7 +8,13 @@ add_custom_target(unit_tests
 add_custom_target(build_system_tests)
 add_custom_target(system_tests
     COMMAND ctest
-            --output-on-failure --exclude-regex "^\\(ut.+\\|python_tests\\)$"
+            --output-on-failure --exclude-regex "^\\(ut.+\\|python_tests\\|dorie_example.+\\)$"
+)
+
+# Add target for manual examples
+add_custom_target(example_tests
+    COMMAND ctest
+            --output-on-failure --tests-regex ^dorie_example.+$
 )
 
 # create the mapping datafile as preparation for all tests
@@ -20,9 +26,9 @@ add_dependencies(unit_tests prepare_testing)
 
 # Create a fake library target to satisfy dune-testtools
 add_library(dorie_test UNKNOWN IMPORTED)
-set_property(TARGET dorie_test
-             PROPERTY IMPORTED_LOCATION ${PROJECT_BINARY_DIR}/activate)
-
+add_library(dorie_example UNKNOWN IMPORTED)
+set_target_properties(dorie_test dorie_example
+    PROPERTIES IMPORTED_LOCATION ${PROJECT_BINARY_DIR}/activate)
 #
 # .. cmake_function:: add_coverage_links
 #

doc/cookbook/1-infiltration-sand/CMakeLists.txt

 
-dorie_add_metaini_test(TARGET dorie_test
+dorie_add_metaini_test(TARGET dorie_example
                        METAINI tutorial-1.mini.in)
 
 configure_file(${CMAKE_CURRENT_BINARY_DIR}/tutorial-1.ini

doc/cookbook/3-random-field/CMakeLists.txt

+# Expand a metaini file into the current binary dir
+function(expand_metaini)
+    foreach(metaini ${ARGN})
+        dune_execute_process(
+            COMMAND ${PROJECT_BINARY_DIR}/run-in-dune-env
+                    dune_expand_metaini.py
+                    --cmake
+                    --ini ${metaini}
+                    --dir ${CMAKE_CURRENT_BINARY_DIR}
+                    --file ${CMAKE_CURRENT_BINARY_DIR}/interface.log
+            ERROR_MESSAGE "Error expanding ${metaini}"
+        )
+    endforeach()
+endfunction()
+
+dorie_add_metaini_test(TARGET dorie_example
+                       METAINI config-miller.mini.in)
+dorie_add_metaini_test(TARGET dorie_example
+                       METAINI config-het.mini.in)
+
+# Configure file paths in the config files
+configure_file(parfield-miller.mini.in parfield-miller.mini)
+configure_file(parfield-het.mini.in parfield-het.mini)
+
+# Expand the meta-ini files into current binary dir
+expand_metaini(${CMAKE_CURRENT_BINARY_DIR}/parfield-miller.mini
+               ${CMAKE_CURRENT_BINARY_DIR}/parfield-het.mini
+)
+
+# Copy inis into source dir for Sphinx to read
+file(COPY ${CMAKE_CURRENT_BINARY_DIR}/3-config-miller.ini
+          ${CMAKE_CURRENT_BINARY_DIR}/3-parfield-miller.ini
+          ${CMAKE_CURRENT_BINARY_DIR}/3-config-heterogeneous.ini
+          ${CMAKE_CURRENT_BINARY_DIR}/3-parfield-heterogeneous.ini
+     DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
+# Copy files required for example runs into build dir
+file(COPY richards_param_miller.yml
+     DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+# Add PFG calls as tests
+_add_test(NAME dorie_example_3_pfg_miller
+         COMMAND ${PROJECT_BINARY_DIR}/run-in-dune-env
+                 dorie pfg ${CMAKE_CURRENT_BINARY_DIR}/3-parfield-miller.ini)
+_add_test(NAME dorie_example_3_pfg_het
+          COMMAND ${PROJECT_BINARY_DIR}/run-in-dune-env
+                  dorie pfg ${CMAKE_CURRENT_BINARY_DIR}/3-parfield-heterogeneous.ini)
+
+# Call "dorie create" before actual tests
+_add_test(NAME dorie_example_3_create
+          COMMAND ${PROJECT_BINARY_DIR}/run-in-dune-env
+                  dorie create)
+
+# Test dependency tree
+set_tests_properties(dorie_example_3_create PROPERTIES FIXTURES_SETUP dorie_3)
+
+set_tests_properties(dorie_example_3_pfg_miller PROPERTIES FIXTURES_SETUP dorie_3_miller)
+set_tests_properties(dorie_example_3-config-miller PROPERTIES FIXTURES_REQUIRED "dorie_3;dorie_3_miller")
+
+set_tests_properties(dorie_example_3_pfg_het PROPERTIES FIXTURES_SETUP dorie_3_het)
+set_tests_properties(dorie_example_3-config-heterogeneous PROPERTIES FIXTURES_REQUIRED "dorie_3;dorie_3_het")

doc/cookbook/3-random-field/config-het.mini.in

+include ${CMAKE_BINARY_DIR}/doc/default_files/config.ini
+
+_test_command = run
+__name = 3-config-heterogeneous
+
+# WARNING: Any change in the line ordering of this file will affect the
+#          tutorial view on this file
+
+[grid.mapping]
+file = mapping.h5
+volume = index_map
+
+[richards.output]
+fileName = heterogeneous
+outputPath = heterogeneous

doc/cookbook/3-random-field/config-miller.mini.in

+include ${CMAKE_BINARY_DIR}/doc/default_files/config.ini
+
+_test_command = run
+__name = 3-config-miller
+
+# WARNING: Any change in the line ordering of this file will affect the
+#          tutorial view on this file
+
+[richards.output]
+fileName = miller
+outputPath = miller
+
+[richards.parameters]
+file = richards_param_miller.yml

doc/cookbook/3-random-field/parfield-het.mini.in

+include ${CMAKE_BINARY_DIR}/doc/default_files/parfield.ini
+
+_test_command = pfg
+__name = 3-parfield-heterogeneous
+
+# WARNING: Any change in the line ordering of this file will affect the
+#          tutorial view on this file
+
+[general]
+outputFile = mapping.h5
+dataset = index_map
+converter = binary
+
+[converter.binary]
+indices = 0 1

doc/cookbook/3-random-field/parfield-miller.mini.in

+include ${CMAKE_BINARY_DIR}/doc/default_files/parfield.ini
+
+_test_command = pfg
+__name = 3-parfield-miller
+
+# WARNING: Any change in the line ordering of this file will affect the
+#          tutorial view on this file
+
+[general]
+outputFile = field.h5
+dataset = miller
+converter = exponential
+
+[converter.exponential]
+varianceScaling = true

doc/cookbook/3-random-field/richards_param_miller.yml

+volumes:
+  sand:
+    index: 0
+    richards:
+      type: MvG
+      parameters:
+        alpha: -2.3
+        n: 4.17
+        k0: 2.2E-5
+        theta_r: 0.03
+        theta_s: 0.31
+        tau: -1.1
+
+scaling:
+  type: Miller
+  data:
+    scale_miller:
+      file: field.h5
+      dataset: miller
+      interpolation: nearest

doc/cookbook/3-random-field/tutorial.rst

+**********************
+Random Field Generator
+**********************
+
+DORiE includes the powerful Gaussian random field generator implemented in
+`dune-randomfield <https://gitlab.dune-project.org/oklein/dune-randomfield>`_.
+We will use this generator for two different use cases to conduct simulations
+on heterogeneous media.
+
+Basics
+------
+
+The random field generator creates a set of random numbers on a physical
+domain. The random variable :math:`X` itself follows a Gaussian (normal)
+distribution with mean :math:`\mu = 0` and a selectable variance
+:math:`\sigma^2`,
+
+.. math::
+
+    X \sim \mathcal{N}(\mu = 0, \sigma^2)
+
+The spatial distribution is determined by covariance functions and correlation
+lengths. Finally, the random number generation itself is pseudo-random, meaning
+that it is actually deterministic but appears random with respect to most
+benchmarks. The seed of the generator therefore lets one replicate a random
+field from an unchanged system configuration.
+
+The random field generator is wrapped into a DORiE Python module which obtains
+the field supplied by the generator and can apply several transformations to
+it, which are tailored for specific use cases. The module is dubbed "Parameter
+Field Generator" (PFG) and takes an INI-configuration file (default:
+``parfield.ini``) as argument, whose keys are documented in
+:doc:`/manual/config-file`. The module is called via the
+:doc:`/manual/cli`.
+
+Use Case: Miller-Similar Medium
+-------------------------------
+
+Small-scale heterogeneity in soil media are typically realized via scalings of
+the parameterization functions, rather than changes of the individual
+parameters themselves. In this section, we will apply
+:ref:`Miller scaling <man-miller_scaling>` to a otherwise homogeneous medium.
+This scaling is based on geometric similarity: It assumes that the local soil
+heterogeneity can be described by a varying lengthscale of the soil grains
+and pores.
+
+Field Generation
+^^^^^^^^^^^^^^^^
+
+We will alter the work flow of the first tutorial by executing the parameter
+field generator (PFG) module in the
+:ref:`command line interface <man-cli-commands>` before starting the solver.
+The default PFG configuration file is called ``parfield.ini``. Enter a
+clean directory and create it together with the other default input files
+by using the ``dorie create`` command.
+
+The PFG module writes a dataset into an H5 file. The file may already exist,
+in which case a new dataset will be added to it. If a dataset with the intended
+name already exists, the script throws an error. Let's choose ``miller`` as
+dataset name for the dataset containing the Miller scaling factors:
+
+.. literalinclude:: parfield-miller.mini.in
+   :language: ini
+   :lines: 9-11
+   :emphasize-lines: 3
+   :caption: parfield.ini
+
+The scaling factors of typical Miller-similar media are log-normally
+distributed. A log-normal distribution is readily obtained from a normal
+distribution by applying the exponential function to all values. This is
+achieved in the PFG module by applying the ``exponential`` converter to the
+data:
+
+.. literalinclude:: parfield-miller.mini.in
+   :language: ini
+   :lines: 9,12
+   :emphasize-lines: 2
+   :caption: parfield.ini
+
+Additionally, the exponential converter can apply a scaling to the resulting
+distribution based on the variance of the original normal distribution. This
+scaling is required to obtain the same mean conductivity as for the
+homogeneous medium. The total transformation reads
+
+.. math::
+
+    f = \exp (x - \sigma^2),
+
+where :math:`f` is the final dataset value and :math:`x` the corresponing value
+of the Gaussian random field. The variance scaling can be enabled in the PFG
+config file:
+
+.. literalinclude:: parfield-miller.mini.in
+   :language: ini
+   :lines: 14-15
+   :emphasize-lines: 2
+   :caption: parfield.ini
+
+Execute the field generator via
+
+.. code-block:: bash
+
+    dorie pfg parfield.ini
+
+Simulation
+^^^^^^^^^^
+
+We now have to insert the resulting field into the simulation. The
+:ref:`parameterization file <man-parameter_file>` of the Richars model
+(default name: ``richards_param.yml``) contains a ``scaling`` section which
+defaults to ``none``. Insert the YAML template for the
+:ref:`Miller scaling <man-miller_scaling>` and specify the file path and
+dataset path/name. In our current example, we use the default grid of
+:math:`100 \times 100` cells on a :math:`1 \text{m} \times 1 \text{m}` domain
+in both the PFG and the simulation, so we can set the interpolation to
+``nearest``, and leave out ``offset`` and ``scale`` completely:
+
+.. literalinclude:: richards_param_miller.yml
+   :language: yaml
+   :lines: 14-20
+   :emphasize-lines: 2-7
+   :caption: richards_param.yml
+
+Now execute the simulation:
+
+.. code-block:: bash
+
+    dorie run config.ini
+
+The following screenshot shows the saturated conductivity field (left) and the
+flux magnitude after 15 time steps (right).
+
+.. image:: https://ts-gitlab.iup.uni-heidelberg.de/dorie/dorie/wikis/uploads/1e2f4028dc1ed54ccaf3d0f9d477911c/miller.png
+   :alt: ParaView screenshot of simulation result.
+
+.. admonition:: Input files
+
+  =================  ==========================================================
+  PFG Configuration  :download:`parfield.ini <3-parfield-miller.ini>`
+  Configuration      :download:`config.ini <3-config-miller.ini>`
+  Parameters         :download:`richards_param.yml <richards_param_miller.yml>`
+  =================  ==========================================================
+
+Use Case: Heterogeneous Medium
+------------------------------
+
+In contrast to the continuous approach of Miller-similarity, soil architecture
+can feature sharp material interfaces at the scale of interest. To produce
+random structures with this quality, we can use the parameter field generator
+(PFG) module to build a :ref:`mapping dataset <man-grid_mapping-dataset>` from
+a random field. Every integer entry of this dataset will then determine the
+parameter set of the medium at the respective grid cell.
+
+Field Generation
+^^^^^^^^^^^^^^^^
+
+To clarify what the resulting field is used for, set the output file and
+dataset names,
+
+.. literalinclude:: parfield-het.mini.in
+   :language: ini
+   :lines: 9-11
+   :emphasize-lines: 2-3
+   :caption: parfield.ini
+
+To compute the mapping dataset from the Gaussian random field for two media
+with indices :math:`i_a` and :math:`i_b`, respectively, the transformation is
+achieved by "cutting the distribution in half,"
+
+.. math::
+
+    f =
+    \begin{cases}
+        i_a \quad \text{if } x > 0, \\
+        i_b \quad \text{else}.
+    \end{cases}
+
+Notice that the ``variance`` setting has therefore no effect on the resulting
+distribution. The converter for this and the indices can be readily selected in
+the PFG configuration file:
+
+.. literalinclude:: parfield-het.mini.in
+   :language: ini
+   :lines: 9,12,13-15
+   :emphasize-lines: 2,5
+   :caption: parfield.ini
+
+Execute the PFG module,
+
+.. code-block:: bash
+
+    dorie pfg parfield.ini
+
+Simulation
+^^^^^^^^^^
+
+In the simulation configuration file (default: ``config.ini``), select the
+mapping dataset inside the output H5 file to insert the medium mapping:
+
+.. literalinclude:: config-het.mini.in
+   :language: ini
+   :lines: 9-11
+   :emphasize-lines: 2-3
+   :caption: config.ini
+
+Now make sure that the indices chosen for the converter in the ``parfield.ini``
+have valid parameter set entries in the
+:ref:`parameterization file <man-parameter_file>` of the Richards model. By
+default, index 0 maps to ``sand`` and index 1 maps to ``silt``:
+
+.. literalinclude:: ../../default_files/richards_param.yml
+   :emphasize-lines: 2-3,14-15
+   :language: yaml
+   :caption: richards_param.yml
+
+Execute the simulation:
+
+.. code-block:: bash
+
+    dorie run config.ini
+
+The following screenshot shows the water content (left) and the flux magnitude
+(right) after 14 time steps.
+
+.. image:: https://ts-gitlab.iup.uni-heidelberg.de/dorie/dorie/wikis/uploads/8ddc2e24341b21100aca6a1037009893/heterogeneous.png
+   :alt: ParaView screenshot of simulation result.
+
+.. admonition:: Input files
+
+  =================  ==================================================================
+  PFG Configuration  :download:`parfield.ini <3-parfield-heterogeneous.ini>`
+  Configuration      :download:`config.ini <3-config-heterogeneous.ini>`
+  Parameters         :download:`richards_param.yml </default_files/richards_param.yml>`
+  =================  ==================================================================

doc/cookbook/CMakeLists.txt

 message(STATUS "Handling cookbook tests")
 
 add_subdirectory(1-infiltration-sand)
+add_subdirectory(3-random-field)

doc/index.rst

@@ -36,6 +36,7 @@ assignment and increment are based on the :doc:`public-api`.
    cookbook/index
    cookbook/1-infiltration-sand/tutorial
    cookbook/2-paraview/tutorial
+   cookbook/3-random-field/tutorial
    cookbook/restart
 
 .. toctree::

doc/manual/parameter-file.rst

@@ -517,6 +517,8 @@ coordinates in the respective spatial dimensions.
     omitted. Only omitting the respective *values* of both keys will lead to
     a parser error.
 
+.. _man-miller_scaling:
+
 Miller Scaling
 """"""""""""""