.gitlab-ci.yml 8.68 KB
Newer Older
Dion Haefner's avatar
Dion Haefner committed
1 2
variables:
  DUNE_CONTROL_PATH: /opt/dune:$CI_PROJECT_DIR
3
  DUNECONTROL: ./dune-common/bin/dunecontrol
4
  CPUS_MULTICORE: 4
5
  CPUS_DIND: 2
6 7
  DOCKER_LOGIN:
    docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW
8 9 10 11 12
  
  # DUNE environment image
  BASE_IMAGE: dorie/dune-env
  # Use semantic versioning (not the version of DUNE) and bump according to
  # to whether changes are backwards-compatible or not.
13
  IMAGE_VERSION: "1.3"
14 15
  DUNE_ENV_IMAGE: ${BASE_IMAGE}:img-v${IMAGE_VERSION}

16 17 18
  CMAKE_FLAGS:
    -DDUNE_PYTHON_VIRTUALENV_SETUP=True
    -DDUNE_PYTHON_ALLOW_GET_PIP=True
19 20
  # Recursively load submodules by default
  GIT_SUBMODULE_STRATEGY: recursive
21 22
  MAKE_FLAGS:
    -j $CPUS_MULTICORE
23
  RUN_IN_DUNE_ENV: $CI_PROJECT_DIR/build-cmake/run-in-dune-env
24

25 26 27 28 29 30
  # Documentation server configuration
  DOC_HOST: root@hermes.iup.uni-heidelberg.de
  DOC_PORT: 2345      # ... forwards to utopia_doc_server container. root ok ;)
  DOC_REMOTE_BASE_DIR: /var/dorie_doc
  DOC_REMOTE_PATH: $DOC_REMOTE_BASE_DIR/$CI_COMMIT_REF_SLUG

31
image: $DUNE_ENV_IMAGE
Dion Haefner's avatar
Dion Haefner committed
32 33 34

before_script:
  - cd /opt/dune
Dion Haefner's avatar
Dion Haefner committed
35

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
# Provide SSH access via the SSH_PRIVATE_KEY and SSH_KNOWN_HOSTS variables
.ssh-access: &ssh_access
  before_script:
    # Run ssh-agent (inside the build environment)
    - eval $(ssh-agent -s)

    # Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
    # We're using tr to fix line endings which makes ed25519 keys work
    # without extra base64 encoding.
    # https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null

    # Create the SSH directory and give it the right permissions
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh

    # Add the known hosts lists to ensure this ssh connection is the right one
    - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts


Dion Haefner's avatar
Dion Haefner committed
57
stages:
58
  - setup
Lukas Riedel's avatar
Lukas Riedel committed
59
  - prep
60
  - build
61
  - test
62 63
  - deploy

64 65

# --- Preparation of Docker Images ---
66
setup:dune-env: &setup
67
  stage: setup
68
  only:
69 70
    variables:
      - $REBUILD_BASE_IMAGE # must be defined and have non-empty value
71 72 73 74 75 76 77
  tags:
    - dind
  image: docker:stable
  services:
    - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
78
  before_script:
79
    - $DOCKER_LOGIN
80
  script:
81
    - docker build -f docker/dune-env.dockerfile
82
      --build-arg PROCNUM=$CPUS_DIND
83
      -t $DUNE_ENV_IMAGE .
84
    - docker push $DUNE_ENV_IMAGE
85 86 87 88 89 90

setup:dune-env-clang:
  <<: *setup
  script:
    - docker build -f docker/dune-env.dockerfile
      --build-arg CC=clang
91
      --build-arg CXX=clang++
92 93 94
      --build-arg PROCNUM=$CPUS_DIND
      -t ${DUNE_ENV_IMAGE}-clang .
    - docker push ${DUNE_ENV_IMAGE}-clang
Dion Haefner's avatar
Dion Haefner committed
95

96
prep:update-dune: &update
97
  <<: *setup
98
  stage: prep
99
  allow_failure: true
100 101 102
  only:
    - master
    - tags
103
  script:
104
    - docker build -f docker/dune-env-update.dockerfile
105 106
      --build-arg DUNE_ENV_IMAGE=$DUNE_ENV_IMAGE
      --build-arg PROCNUM=$CPUS_DIND
107
      -t $DUNE_ENV_IMAGE .
108
    - docker push $DUNE_ENV_IMAGE
109

110 111 112 113
prep:update-dune-clang:
  <<: *update
  script:
    - docker build -f docker/dune-env-update.dockerfile
114
      --build-arg DUNE_ENV_IMAGE=${DUNE_ENV_IMAGE}-clang
115 116
      --build-arg CC=clang
      --build-arg CXX=clang++
117
      --build-arg PROCNUM=$CPUS_DIND
118 119
      -t ${DUNE_ENV_IMAGE}-clang .
    - docker push ${DUNE_ENV_IMAGE}-clang
120 121 122


# --- Build jobs ---
123
build:system-tests: &build-tests
124
  stage: build
Dion Haefner's avatar
Dion Haefner committed
125
  script:
Lukas Riedel's avatar
Lukas Riedel committed
126
    - CMAKE_FLAGS="$CMAKE_FLAGS"
127
      $DUNECONTROL --only=dorie configure
128
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS dorie-rfg
129
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_system_tests
130
  artifacts:
131
    name: "$CI_JOB_NAME"
132
    paths:
133
      - $CI_PROJECT_DIR/build-cmake
134
    expire_in: 1 day
135

136 137 138 139 140 141
build:unit-tests:
  <<: *build-tests
  script:
    - CMAKE_FLAGS="$CMAKE_FLAGS
                   -DCMAKE_BUILD_TYPE=Debug"
      $DUNECONTROL --only=dorie configure
142
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_unit_tests
143

144
build:debug: &debug
145
  stage: build
146
  allow_failure: true
147
  script:
Lukas Riedel's avatar
Lukas Riedel committed
148 149
    - CMAKE_FLAGS="
        $CMAKE_FLAGS
150
        -DCMAKE_BUILD_TYPE=Debug
Lukas Riedel's avatar
Lukas Riedel committed
151
        -DCMAKE_CXX_FLAGS_DEBUG='-Werror'"
152
      $DUNECONTROL --only=dorie configure
153
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS dorie-rfg
154
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_unit_tests
155
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS richards_d2_r1 transport_d2_r0_t0
156

157 158 159
build:debug-clang:
  <<: *debug
  image: ${DUNE_ENV_IMAGE}-clang
160 161 162
  script:
    - CMAKE_FLAGS="
        $CMAKE_FLAGS
163
        -DCMAKE_BUILD_TYPE=Debug
164 165 166 167
        -DCMAKE_C_COMPILER=clang
        -DCMAKE_CXX_COMPILER=clang++
        -DCMAKE_CXX_FLAGS_DEBUG='-Werror'"
      $DUNECONTROL --only=dorie configure
168
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS dorie-rfg
169
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_unit_tests
170
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS transport_d2_r0_t0 transport_d2_r1_t0
171

172 173 174 175 176 177 178 179 180 181 182
build:docs:
  stage: build
  script:
    - CMAKE_FLAGS="$CMAKE_FLAGS"
      $DUNECONTROL --only=dorie configure
    - $DUNECONTROL --only=dorie make doc
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
      - $CI_PROJECT_DIR/build-cmake/doc
    expire_in: 1 day
183 184

# --- Tests ---
185
test:system-tests: &test
186 187
  stage: test
  dependencies:
188
    - build:system-tests
189
  script:
Lukas Riedel's avatar
Lukas Riedel committed
190
    - $DUNECONTROL --only=dorie configure
191
    - $DUNECONTROL --only=dorie make system_tests
192 193 194
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
195
      - $CI_PROJECT_DIR/build-cmake/test
196
    expire_in: 1 day
197

198
test:unit-tests:
199
  <<: *test
200 201
  dependencies:
    - build:unit-tests
202
  script:
203
    # install coverage reporter
204 205
    - $DUNECONTROL --only=dorie bexec
      "./run-in-dune-env python3 -m pip install gcovr"
206
    # perform the tests
207
    - $DUNECONTROL --only=dorie configure
208 209
    - $DUNECONTROL --only=dorie make unit_tests
    # report coverage
210
    - cd $CI_PROJECT_DIR/build-cmake
211
    - mkdir -p dune/dorie/test/coverage
212 213
    - ./run-in-dune-env gcovr
      --root ../
214
      --exclude ../dune/dorie/test
215
      --exclude ../plugins
216
      --html --html-details
217
      -o dune/dorie/test/coverage/coverage.html
218
    # run again for coverage report in GitLab
219 220
    - ./run-in-dune-env gcovr
      --root ../
221
      --exclude ../dune/dorie/test
222
      --exclude ../plugins
223
  coverage: '/^TOTAL.*\s+(\d+\%)$/'
224 225 226 227 228
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
      - $CI_PROJECT_DIR/build-cmake/dune/dorie/test
    expire_in: 1 day
229

230 231 232 233 234 235 236
test:python-tests:
  <<: *test
  dependencies: []
  script:
    - $DUNECONTROL --only=dorie configure
    - $DUNECONTROL --only=dorie make test_python

237
# --- Deploy jobs ---
238
deploy:dockerhub-devel: &deploy
Lukas Riedel's avatar
Lukas Riedel committed
239
  stage: deploy
240
  only:
241
    - master@dorie/dorie
242 243 244 245 246 247 248
  tags:
    - dind
  image: docker:stable
  services:
    - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
249
  dependencies: []
250
  before_script:
251
    - $DOCKER_LOGIN
252
  script:
253
    - docker build -f docker/dorie.dockerfile
254
      --build-arg BASE_IMG_VERSION=$IMAGE_VERSION --build-arg PROCNUM=$CPUS_DIND
255
      -t dorie/dorie:devel .
256
    - docker push dorie/dorie:devel
257 258 259
  environment:
    name: docker/devel
    url: https://hub.docker.com/r/dorie/dorie
260 261

deploy:dockerhub-stable:
262
  <<: *deploy
263
  only:
264
    - tags@dorie/dorie
265
  script:
266
    - docker build -f docker/dorie.dockerfile
267
      --build-arg BASE_IMG_VERSION=$IMAGE_VERSION --build-arg PROCNUM=$CPUS_DIND
268
      -t dorie/dorie:$CI_COMMIT_TAG .
269
    - docker push dorie/dorie:$CI_COMMIT_TAG
270 271 272
  environment:
    name: docker/$CI_COMMIT_TAG
    url: https://hub.docker.com/r/dorie/dorie
273

274
deploy:docs:
275
  stage: deploy
276
  only:
277
    - branches@dorie/dorie
278
  dependencies:
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
    - build:docs
  needs: ["build:docs"]

  <<: *ssh_access
  script:
    # Create the directory on the remote, removing any prior version
    - echo "Creating remote directory $DOC_REMOTE_PATH ..."
    - ssh -p $DOC_PORT $DOC_HOST "rm -rf $DOC_REMOTE_PATH"
    - ssh -p $DOC_PORT $DOC_HOST "mkdir -p $DOC_REMOTE_PATH/doxygen/"

    # Copy sphinx & doxygen HTML documentation to remote
    - cd build-cmake/doc
    - echo "Uploading documentation to $DOC_REMOTE_PATH/ ..."
    - scp -P $DOC_PORT -pr html         $DOC_HOST:$DOC_REMOTE_PATH/
    - scp -P $DOC_PORT -pr doxygen/html $DOC_HOST:$DOC_REMOTE_PATH/doxygen/

  environment:
    name: docs/$CI_COMMIT_REF_NAME
    url: https://hermes.iup.uni-heidelberg.de/dorie_doc/$CI_COMMIT_REF_SLUG/html/
    on_stop: deploy:stop_docs

# This job is called when the environment is stopped, which automatically
# happens when the respective branch is deleted
deploy:stop_docs:
  stage: deploy
  when: manual
  variables:
    # Stop GitLab from checking out the commit again (branch is deleted)
    GIT_STRATEGY: none
  dependencies: []
  needs: ["build:docs"]

  <<: *ssh_access
  script:
    - echo "Removing remote directory $DOC_REMOTE_PATH ..."
    - ssh -p $DOC_PORT $DOC_HOST "rm -rf $DOC_REMOTE_PATH"

316
  environment:
317 318
    name: docs/$CI_COMMIT_REF_NAME
    action: stop