Commit a8a3dce4 authored by Lukas Riedel's avatar Lukas Riedel

Merge branch '104-re-bundle-python-packages' into 'master'

Resolve "Re-bundle Python packages"

Closes #104

See merge request !100
parents 56d5f506 fa9e0be9
......@@ -2,9 +2,8 @@
build-cmake/
# Exclude generated files
python/parfield/wrapper/pf_from_file.py
python/testtools/wrapper/test_dorie.py
python/testtools/wrapper/test_dorie_pfg.py
python/dorie/wrapper/pf_from_file.py
python/dorie/wrapper/test_dorie.py
# Ignore temporary and auto-generated files #
*~
......
......@@ -58,6 +58,7 @@
* The `build` CI stage now explicitly builds system tests and unit tests. The
job `build:main` has been removed. The `build:debug` jobs now build the
unit test targets.
* Merge Python packages into single new package `dorie`.
### Deprecated
* The configuration file key `[parameters.interpolation]` is deprecated due to
......
......@@ -30,7 +30,7 @@ The dorie.testtools scripts
**Parameter keys**:
* ``_evaluation``: Specifies the evaluation script that is to be called. Can be the name of any module in :mod:`dorie.testtools.evaluation`.
* ``__inifile_optionkey``: The argument that is passed to the DORiE executable before the parameter file. This should be `run`` in most situations.
* ``__inifile_optionkey``: The argument that is passed to the DORiE executable before the parameter file. This should be `run` in most situations.
All command line arguments are passed through the dune-testtools CMake machinery and parsed by the
:mod:`dune.testtools` python package.
......@@ -39,41 +39,52 @@ The dorie.testtools scripts
The dorie.testtools core modules
================================
.. automodule:: dorie.testtools
The module is intended to be called via the internal wrapper
:mod:`dorie.testtools.wrapper.test_dorie.py`.
dorie.testtools consists of two submodules:
.. automodule:: dorie.testtools.wrapper.test_dorie
:members:
:undoc-members:
dorie.testtools consists of multiple submodules:
- :mod:`dorie.testtools.evaluation`, containing all modules responsible for the evaluation of
the system tests
- :mod:`dorie.testtools.utilities`, which contains convenience functions that are included by
the evaluation modules and are too specific to be contained in :mod:`dorie.utilities`
- Evaluation modules for the system tests, where each module corresponds to one
of the supported commands of the :mod:`dorie` wrapper script.
The evaluation module
+++++++++++++++++++++
The ``dorie_create`` module
+++++++++++++++++++++++++++
.. automodule:: dorie.testtools.evaluation
.. automodule:: dorie.testtools.dorie_create.compare
:members:
:undoc-members:
Convergence evaluator
----------------------
The ``dorie_pfg`` module
++++++++++++++++++++++++
.. automodule:: dorie.testtools.evaluation.convergence
.. automodule:: dorie.testtools.dorie_pfg.reference
:members:
:undoc-members:
Correlation evaluator
----------------------
The ``dorie_plot`` module
+++++++++++++++++++++++++
.. automodule:: dorie.testtools.evaluation.correlation
Dummy module without actual evaluation scripts.
The ``dorie_run`` module
++++++++++++++++++++++++
.. automodule:: dorie.testtools.dorie_run.muphi
:members:
:undoc-members:
ODE evaluator
--------------
.. automodule:: dorie.testtools.dorie_run.ode
:members:
:undoc-members:
.. automodule:: dorie.testtools.evaluation.ode
.. automodule:: dorie.testtools.dorie_run.reference
:members:
:undoc-members:
......@@ -84,6 +95,13 @@ The utilities module
:members:
:undoc-members:
The MuPhi output file reader
----------------------------
.. automodule:: dorie.testtools.utilities.muphi_reader
:members:
:undoc-members:
Richards equation definitions
------------------------------
......
# install the following modules in editable mode: 'pip -e'
set(DUNE_PYTHON_INSTALL_EDITABLE TRUE)
# install required python modules
dune_python_install_package(PATH utilities)
dune_python_install_package(PATH parscraper)
add_subdirectory(parfield/wrapper)
dune_python_install_package(PATH parfield)
# install testing module
if(dune-testtools_FOUND)
add_subdirectory(testtools/wrapper)
dune_python_install_package(PATH testtools)
endif()
# install Python package
add_subdirectory(dorie/wrapper)
dune_python_install_package(PATH dorie)
# set cache variable
set(DORIE_PYTHON_MODULES ${python_paths} CACHE PATH "Paths to the python modules")
#!/usr/bin/env python3
from setuptools import setup, find_packages
setup(name='dorie',
version='0.1',
namespace_packages=['dorie'],
description='Python package providing the DORiE parameter scraper',
author='Lukas Riedel',
author_email='dorieteam@iup.uni-heidelberg.de',
url='https://ts-gitlab.iup.uni-heidelberg.de/dorie/dorie',
packages=find_packages(),
install_requires= ['cycler',
'configparser',
'h5py',
'matplotlib',
'numpy',
'Pillow',
'pyparsing',
'python-dateutil',
'pytz',
'scipy',
'six',
'wheel'
],
scripts=[
'wrapper/pf_from_file.py',
'wrapper/plot_vtk.py',
'wrapper/scrape_folder.py',
'wrapper/test_dorie.py'
]
)
configure_file(pf_from_file.py.in ${CMAKE_CURRENT_SOURCE_DIR}/pf_from_file.py)
configure_file(test_dorie.py.in ${CMAKE_CURRENT_SOURCE_DIR}/test_dorie.py)
\ No newline at end of file
recursive-include dorie/parscraper *.py
#!/usr/bin/env python3
import sys
from setuptools import setup
def randomfield_scripts():
return ['wrapper/pf_from_file.py']
setup(name='dorie.parfield',
version='0.1',
namespace_packages=['dorie'],
description='Python package providing the DORiE parameter field tools',
author='Dion Häfner <mail@dionhaefner.de>',
author_email='dorieteam@iup.uni-heidelberg.de',
url='',
install_requires=['dorie.utilities',
'h5py',
'Pillow',
'configparser'],
packages = ['dorie.parfield',
'dorie.parfield.converter',
'dorie.parfield.parameterization'],
scripts=randomfield_scripts()
)
recursive-include dorie/parscraper *.py
__import__('pkg_resources').declare_namespace(__name__)
#!/usr/bin/env python3
import sys
from setuptools import setup
def dorie_parscraper_scripts():
return ['./wrapper/scrape_folder.py']
setup(name='dorie.parscraper',
version='0.1',
namespace_packages=['dorie'],
description='Python package providing the DORiE parameter scraper',
author='Dion Häfner <mail@dionhaefner.de>',
author_email='dorieteam@iup.uni-heidelberg.de',
url='',
packages= ['dorie.parscraper',
'dorie.parscraper.wrapper',
'dorie.parscraper.writers',
'dorie.parscraper.readers'],
install_requires= ['cycler',
'matplotlib',
'numpy',
'pyparsing',
'python-dateutil',
'pytz',
'scipy',
'six',
'wheel'
],
scripts=dorie_parscraper_scripts()
)
recursive-include dorie/testtools *.py
__import__('pkg_resources').declare_namespace(__name__)
import os
from scipy import integrate, interpolate
import matplotlib as mpl
mpl.use("agg")
import matplotlib.pyplot as plt
import numpy as np
import h5py
from dorie.testtools.utilities.decorators import evaluation_script
from dorie.testtools.utilities import statistics
from dorie.utilities.text_to_bool import text_to_bool
@evaluation_script
def evaluate(iniinfo,runtime):
h5_path = iniinfo["generator.fft.outputPath"] + "/yfield.stoch.h5"
with h5py.File(h5_path, 'r') as f:
y_data = np.array(f.get("/stochastic"), dtype=np.float64)
print("Operating on file: {}".format(h5_path))
# Calculate autocorrelation and correlation lengths
ac = statistics.autocorrelation(y_data)
rank = y_data.ndim
dx = np.array(iniinfo["generator.extensions"].split(), dtype=np.float) / y_data.shape
assert(len(dx) == rank)
cl = [integrate.simps(ac,dx=dxi,axis=i).flat[0] for i, dxi in enumerate(dx)][::-1]
# Compare to given values
cl_ini = iniinfo["generator.fft.correlationLengths"].split()
cl_ini = list(map(float,cl_ini))
if "_correlation.abstol" in iniinfo:
abstol = float(iniinfo["_correlation.abstol"])
else:
abstol = 1E-4
if "_correlation.reltol" in iniinfo:
reltol = float(iniinfo["_correlation.reltol"])
else:
reltol = 0.1
passed = np.allclose(cl,cl_ini,rtol=reltol,atol=abstol)
# Print output
print("Measured correlation lengths: " + ", ".join(["{:.2e}".format(c) for c in cl]))
print("Expected correlation lengths: " + ", ".join(["{:.2e}".format(c) for c in cl_ini]))
if passed:
print("- pass\n")
else:
print("- fail\n")
if rank == 2:
plt.imshow(ac,norm=mpl.colors.LogNorm())
plt.colorbar()
plt.savefig("ac-{}.png".format(iniinfo["__name"].split("/")[-1]))
return bool(passed)
import os
from scipy import interpolate
import numpy as np
from dorie.utilities.vtktools import pvd_reader
from dorie.testtools.utilities.decorators import evaluation_script
from dorie.testtools.utilities import statistics
@evaluation_script
def evaluate(iniinfo,runtime):
"""
Assume the grid gets finer with every iteration for times t > mintime
"""
try:
mintime = float(iniinfo["_convergence.minTime"])
except KeyError:
raise ValueError("_convergence.minTime must be present in ini file for convergence evaluator")
pvd = iniinfo["output.fileName"] + ".pvd"
vtks = pvd_reader.read(pvd)
# generator that yields pairwise list entries
def vtk_pairs(vtklist):
n = len(vtklist)
for i in range(n-1):
if vtklist[i].time > mintime:
yield vtklist[i], vtklist[i+1]
passed = True
l2_old = np.inf
for vtk1, vtk2 in vtk_pairs(vtks):
print("Comparing files: {}, {}".format(vtk1,vtk2))
# Process data
grid1 = vtk1.read(["head"])
cells1 = grid1.cellCenters()
data1 = grid1.data()
head1 = data1["head"]
grid2 = vtk2.read(["head"])
cells2 = grid2.cellCenters()
data2 = grid2.data()
head2 = data2["head"]
interpolator1 = interpolate.LinearNDInterpolator(cells1.T, head1)
interpolator2 = interpolate.LinearNDInterpolator(cells2.T, head2)
# create a meshgrid with 30 points in each dimension
reg_grid_space = [np.linspace(k.min(),k.max(),30) for k in cells1]
reg_grid = np.array(np.meshgrid(*reg_grid_space))
n = reg_grid.shape
reg_grid = reg_grid.reshape(n[0],-1)
residual = np.squeeze(interpolator1(reg_grid.T) - interpolator2(reg_grid.T))
residual = residual.reshape(*n[1:])
mask = np.isnan(residual) # for filtering NaN's
residual[mask] = 0
# Calculate L2 residual
l2 = statistics.norm_L2(residual, reg_grid_space)
# Print output
print("L2 error: {0:.2e}\n".format(l2))
if l2_old < l2:
passed = False
break
l2_old = l2
return passed
import numpy as np
from scipy import integrate, signal
def norm_L2(a,x):
a_squared = np.squeeze(a**2)
n = len(np.shape(a_squared))
for i in range(n):
a_squared = integrate.simps(a_squared,x=x[i],axis=0)
return np.sqrt(a_squared)
def autocorrelation(H, pad=None):
"""
Calculates the autocorrelation of an array via ffts.
.. math::
\\text{autocovariance}(H,x) = \\text{fft}^{-1}(|\\text{fft}(H - \\overline{H},k)|^2,x)
\\text{autocorrelation}(H,x) = \\text{autocovariance}(x) / \\text{autocovariance}(0)
:param H: input array
:param pad: Pads each axis of the input with pad[i] zeros before and after each FFT.
"""
#pad = np.array(H.shape) if pad is None else pad
#H = H - np.mean(H)
#ft_signal = np.fft.fftn(H,pad)
#B = np.fft.ifftn(np.abs(ft_signal)**2,pad)
reverse = [slice(n,0,-1) for n in H.shape]
autocovariance = signal.fftconvolve(H,H[reverse],mode="full")[reverse]
return autocovariance / autocovariance.flat[0]
#!/usr/bin/env python3
import sys
from setuptools import setup
def dorie_testtools_scripts():
return ['./wrapper/test_dorie.py']
setup(name='dorie.testtools',
version='0.1',
namespace_packages=['dorie'],
description='Python modules for testing DORiE, powered by dune-testtools',
author='Dion Häfner <mail@dionhaefner.de>',
author_email='dorieteam@iup.uni-heidelberg.de',
url='',
packages=['dorie.testtools',
'dorie.testtools.dorie_create',
'dorie.testtools.dorie_pfg',
'dorie.testtools.dorie_plot',
'dorie.testtools.dorie_run',
'dorie.testtools.utilities',
'dorie.testtools.wrapper'
],
install_requires=['dune.testtools',
'cycler',
'matplotlib',
'numpy',
'pyparsing',
'python-dateutil',
'pytz',
'scipy',
'six',
'wheel'],
scripts=dorie_testtools_scripts())
configure_file(test_dorie.py.in ${CMAKE_CURRENT_SOURCE_DIR}/test_dorie.py)
recursive-include dorie/testtools *.py
__import__('pkg_resources').declare_namespace(__name__)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sys
from setuptools import setup
def dorie_utilities_scripts():
return ["wrapper/plot_vtk.py"]
setup(name='dorie.utilities',
version='0.1',
namespace_packages=['dorie'],
description='Multipurpose utilities used by the other DORiE python extensions',
author='Dion Häfner <mail@dionhaefner.de>',
author_email='dorieteam@iup.uni-heidelberg.de',
url='',
packages=['dorie.utilities',
'dorie.utilities.vtktools'],
install_requires=['cycler',
'matplotlib',
'numpy',
'pyparsing',
'python-dateutil',
'pytz',
'scipy',
'six',
'wheel'],
scripts=dorie_utilities_scripts()
)