Commit ef5f8fdd authored by Lukas Riedel's avatar Lukas Riedel

Split ParameterScraper into two separate commands

'scrape_folder' now dumps the data retrieved into a YAML file.
This file is then loaded by 'write_config' to write the actual config.
parent f7cfafc4
function(scrape_parameters SOURCE_DIR XML_FILE CSS OUTPUT)
function(scrape_parameters)
message(STATUS "Running parameter scraper on sources ${SOURCE_DIR}")
set(OPTIONS MODEL)
cmake_parse_arguments(SCRAPE_PARAMETERS "" "${OPTIONS}" "" ${ARGN})
set(OPTIONS OVERWRITE)
set(SINGLE SOURCE_DIR XML_FILE OUTPUT MODEL)
cmake_parse_arguments(SCRAPE_PARAMETERS
"${OPTIONS}" "${SINGLE}" "" ${ARGN})
# set options list
set(scrape_options
--source ${SCRAPE_PARAMETERS_SOURCE_DIR}
--xml ${SCRAPE_PARAMETERS_XML_FILE}
--out ${SCRAPE_PARAMETERS_OUTPUT}
)
# evaluate options
if(SCRAPE_PARAMETERS_MODEL)
if(DEPLOY_SPHINX_SOURCE_URL)
execute_process(COMMAND ${CMAKE_BINARY_DIR}/run-in-dune-env scrape_folder.py --source ${SOURCE_DIR}
--xml ${XML_FILE} --out ${OUTPUT} --css ${CSS}
--source_url ${DEPLOY_SPHINX_SOURCE_URL}
--model ${SCRAPE_PARAMETERS_MODEL}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
RESULT_VARIABLE RETURN_CODE)
else()
execute_process(COMMAND ${CMAKE_BINARY_DIR}/run-in-dune-env scrape_folder.py --source ${SOURCE_DIR}
--xml ${XML_FILE} --out ${OUTPUT} --css ${CSS} --model ${SCRAPE_PARAMETERS_MODEL}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
RESULT_VARIABLE RETURN_CODE)
endif()
else()
if(DEPLOY_SPHINX_SOURCE_URL)
execute_process(COMMAND ${CMAKE_BINARY_DIR}/run-in-dune-env scrape_folder.py --source ${SOURCE_DIR}
--xml ${XML_FILE} --out ${OUTPUT} --css ${CSS}
--source_url ${DEPLOY_SPHINX_SOURCE_URL}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
RESULT_VARIABLE RETURN_CODE)
else()
execute_process(COMMAND ${CMAKE_BINARY_DIR}/run-in-dune-env scrape_folder.py --source ${SOURCE_DIR}
--xml ${XML_FILE} --out ${OUTPUT} --css ${CSS}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
RESULT_VARIABLE RETURN_CODE)
endif()
list(APPEND scrape_options --model ${SCRAPE_PARAMETERS_MODEL})
endif()
if(SCRAPE_PARAMETERS_OVERWRITE)
list(APPEND scrape_options --overwrite)
endif()
if (NOT ${RETURN_CODE} EQUAL 0)
execute_process(COMMAND ${CMAKE_BINARY_DIR}/run-in-dune-env
scrape_folder.py ${scrape_options}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
RESULT_VARIABLE RETURN_CODE)
if (NOT ${RETURN_CODE} EQUAL 0)
message(FATAL_ERROR "Parameter scraper failed. DORiE can not be built.")
endif()
endif()
endfunction()
function(create_default_config INPUT OUTPUT SOURCE_DIR CSS)
message(STATUS "Creating default config ${OUTPUT}.")
set(options
${INPUT}
${OUTPUT}
--source ${SOURCE_DIR}
--css ${CSS})
execute_process(COMMAND ${CMAKE_BINARY_DIR}/run-in-dune-env
write_config.py ${options}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
RESULT_VARIABLE RETURN_CODE)
if (NOT ${RETURN_CODE} EQUAL 0)
message(FATAL_ERROR "Creating the default config file failed.")
endif()
endfunction()
file(COPY ${CMAKE_CURRENT_LIST_DIR} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/..)
# file(COPY ${CMAKE_CURRENT_LIST_DIR} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/..)
scrape_parameters(
${PROJECT_SOURCE_DIR}/dune/dorie-rfg
${CMAKE_CURRENT_SOURCE_DIR}/field-parameters.xml
${CMAKE_CURRENT_SOURCE_DIR}/parameters.css
"parfield.ini;field-parameters.html;field-parameters.rst")
SOURCE_DIR ${PROJECT_SOURCE_DIR}/dune/dorie-rfg
XML_FILE ${CMAKE_CURRENT_SOURCE_DIR}/field-parameters.xml
OUTPUT rfg.yml
OVERWRITE
)
scrape_parameters(
${PROJECT_SOURCE_DIR}/dune/dorie
${CMAKE_CURRENT_SOURCE_DIR}/common-parameters.xml
${CMAKE_CURRENT_SOURCE_DIR}/parameters.css
"common-config.ini;common-parameters.html;common-parameters.rst")
SOURCE_DIR ${PROJECT_SOURCE_DIR}/dune/dorie
XML_FILE ${CMAKE_CURRENT_SOURCE_DIR}/common-parameters.xml
OUTPUT config.yml
OVERWRITE
)
scrape_parameters(
${PROJECT_SOURCE_DIR}/dune/dorie/model/richards
${CMAKE_CURRENT_SOURCE_DIR}/richards-parameters.xml
${CMAKE_CURRENT_SOURCE_DIR}/parameters.css
"richards-config.ini;richards-parameters.html;richards-parameters.rst"
MODEL "richards")
# Gather different configuration files
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/config.ini.in "####### Common parameters #######\n\n")
file(READ ${CMAKE_CURRENT_BINARY_DIR}/common-config.ini CONTENTS)
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/config.ini.in "${CONTENTS}\n\n")
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/config.ini.in "####### Richards parameters #######\n\n")
file(READ ${CMAKE_CURRENT_BINARY_DIR}/richards-config.ini CONTENTS)
file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/config.ini.in "${CONTENTS}")
SOURCE_DIR ${PROJECT_SOURCE_DIR}/dune/dorie/model/richards
XML_FILE ${CMAKE_CURRENT_SOURCE_DIR}/richards-parameters.xml
OUTPUT config.yml
MODEL "richards"
)
configure_file(${CMAKE_CURRENT_BINARY_DIR}/config.ini.in ${CMAKE_CURRENT_BINARY_DIR}/config.ini COPYONLY)
create_default_config(
${CMAKE_CURRENT_BINARY_DIR}/rfg.yml
"parfield.ini;field-parameters.html;field-parameters.rst"
${PROJECT_SOURCE_DIR}/dune/dorie-rfg
${CMAKE_CURRENT_SOURCE_DIR}/parameters.css
)
create_default_config(
${CMAKE_CURRENT_BINARY_DIR}/config.yml
"config.ini;parameters.html;parameters.rst"
${PROJECT_SOURCE_DIR}/dune/dorie-rfg
${CMAKE_CURRENT_SOURCE_DIR}/parameters.css
)
......@@ -43,7 +43,8 @@ def parse(filename,model=None):
for re_get_match in re_get_matches:
name = re_get_match[2].split(".")
if model:
category = model + "." + ".".join(name[:-1])
category = model + "." + ".".join(name[:-1])
filename = os.path.join(model, filename)
else:
category = ".".join(name[:-1])
key = name[-1]
......
......@@ -3,52 +3,52 @@ import sys
import warnings
import argparse
import traceback
from collections import OrderedDict
from yaml import dump, load
from dorie.parscraper import readers, writers, match_parameters
from dorie.parscraper.warnings import OutputWarning
def scrape(xml_file,source_folder,out,model=None,css=None,debug=False):
# PARSE XML FILE
if os.path.isfile(xml_file):
xml_parameters = readers.xml.parse(xml_file,model)
else:
raise IOError("XML file {0} does not exist".format(xml_file))
# CHECK SOURCE FOLDER
if not source_folder.endswith("/"):
source_folder += "/"
if not os.path.isdir(source_folder):
raise IOError("Source folder {0} does not exist".format(source_folder))
# ITERATE OVER ALL .cc AND .hh FILES IN SOURCE FOLDER, AND CALL SOURCE SCRAPER
source_parameters = []
for subdir, dirs, files in os.walk(source_folder):
for f in files:
if f.endswith(".cc") or f.endswith(".hh"):
full_path = os.path.join(subdir, f)
source_parameters.append(readers.source.parse(full_path,model))
source_parameters = [item for sublist in source_parameters for item in sublist] # flatten list
# MATCH XML AND SCRAPED PARAMETERS
matched_parameters = match_parameters.match(xml_parameters,source_parameters,source_folder)
# CALL OUTPUT SCRIPTS
for o in out:
file_suffix = o.split(".")[-1]
if hasattr(writers,file_suffix):
writer = getattr(writers,file_suffix)
try:
writer.write(matched_parameters,o,source_folder,css=css)
except BaseException as e:
if file_suffix == "ini":
raise
else:
warnings.warn("Output failed for file {0} with error:\n{1}".format(o,repr(e)), OutputWarning)
if debug:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
else:
warnings.warn("Unknown output format: .{}. Skipping output".format(file_suffix), OutputWarning)
from dorie.parscraper.parameter import Parameter
def scrape(xml,
source,
out,
overwrite=False,
model=None,
debug=False):
# PARSE XML FILE
if os.path.isfile(xml):
xml_parameters = readers.xml.parse(xml,model)
else:
raise IOError("XML file {0} does not exist".format(xml))
# CHECK SOURCE FOLDER
if not source.endswith("/"):
source += "/"
if not os.path.isdir(source):
raise IOError("Source folder {0} does not exist".format(source))
# ITERATE OVER ALL .cc AND .hh FILES IN SOURCE FOLDER, AND CALL SOURCE SCRAPER
source_parameters = []
for subdir, dirs, files in os.walk(source):
for f in files:
if f.endswith(".cc") or f.endswith(".hh"):
full_path = os.path.join(subdir, f)
source_parameters.append(readers.source.parse(full_path,model))
source_parameters = [item for sublist in source_parameters for item in sublist] # flatten list
# MATCH XML AND SCRAPED PARAMETERS
matched_parameters = match_parameters.match(xml_parameters,source_parameters,source)
# WRITE YAML DATA FILE
if not overwrite:
# concatenate file contents and current data
with open(out, 'r') as file:
data = load(file)
data.update(matched_parameters)
matched_parameters = data
with open(out, 'w') as file:
file.write(dump(matched_parameters, default_flow_style=False))
......@@ -30,6 +30,7 @@ setup(name='dorie',
'wrapper/pf_from_file.py',
'wrapper/plot_vtk.py',
'wrapper/scrape_folder.py',
'wrapper/test_dorie.py'
'wrapper/test_dorie.py',
'wrapper/write_config.py'
]
)
......@@ -19,14 +19,16 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('-x','--xml',help="The XML file holding parameter meta-information",required=True)
parser.add_argument('-s','--source',help='The C++ source folder to scrape',required=True)
parser.add_argument('-o','--out',help='Output file(s)',required=True,nargs="*")
parser.add_argument('-c','--css',help='Path to a CSS file to include into HTML output',required=False)
parser.add_argument('-o','--out',help='YAML output file',required=True)
parser.add_argument('--overwrite', action="store_true",
help="Overwrite existing output file. "
"Append otherwise.")
parser.add_argument('-m','--model',help='Prefix of the parsed model',required=False)
parser.add_argument('--debug',help='Display warnings',action='store_true',required=False)
args = vars(parser.parse_args())
# CALL WRAPPER
scrape_folder.scrape(args["xml"],args["source"],args["out"],args["model"],args["css"],args["debug"])
scrape_folder.scrape(**args)
# HANDLE WARNINGS
if not "debug" in args:
......
#!/usr/bin/env python3
from __future__ import absolute_import
import os
import warnings
import argparse
from collections import OrderedDict
from yaml import load
from dorie.parscraper import writers
from dorie.parscraper.parameter import Parameter
def write(data, args):
# CALL OUTPUT SCRIPTS
for out in args.output:
file_suffix = out.split(".")[-1]
if hasattr(writers, file_suffix):
writer = getattr(writers, file_suffix)
try:
writer.write(data,
out,
args.source,
css=args.css)
except BaseException as e:
if file_suffix == "ini":
raise
else:
warnings.warn("Output failed for file {0} with error:\n{1}".format(out, repr(e)), OutputWarning)
if args.debug:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
else:
warnings.warn("Unknown output format: .{}. Skipping output".format(file_suffix), OutputWarning)
def read_data(args):
with open(args.input, 'r') as file:
return load(file)
def parse_args():
parser = argparse.ArgumentParser(
description="Load the YAML file containing all scraped parameter "
"data and write this data into other formats.\n"
"Supported formats are INI, HTML, RST.")
parser.add_argument("input", type=os.path.realpath,
help="YAML file containing all parameter data")
parser.add_argument("output", type=os.path.realpath, nargs='+',
help="Output files to write data to.")
parser.add_argument("-s", "--source", type=os.path.realpath,
required=True,
help="Top level path to the source files. "
"Required for appropriate file links.")
parser.add_argument("-c", "--css", required=False,
help="Path to a CSS file to include into HTML output")
parser.add_argument("--debug", action="store_true", required=False,
help="Display warnings")
return parser.parse_args()
if __name__ == "__main__":
args = parse_args()
data = read_data(args)
write(data, args)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment