.gitlab-ci.yml 9.74 KB
Newer Older
Dion Haefner's avatar
Dion Haefner committed
1
variables:
2 3
  DUNE_DIR: /opt/dune
  DUNE_CONTROL_PATH: $DUNE_DIR:$CI_PROJECT_DIR
4
  DUNECONTROL: ./dune-common/bin/dunecontrol
Lukas Riedel's avatar
Lukas Riedel committed
5
  CPUS_MULTICORE: 4
6
  CPUS_DIND: 2
7 8
  DOCKER_LOGIN:
    docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW
9

10
  # DUNE environment image
11 12 13 14 15
  BASE_IMAGE_NAME: dorie/dune-env
  # NOTE: Bump versions if Dockerfiles are updated
  BASE_IMAGE_UBUNTU_VERSION: focal
  BASE_IMAGE_VERSION: "1"
  DUNE_ENV_IMAGE: ${BASE_IMAGE_NAME}:${BASE_IMAGE_UBUNTU_VERSION}-v${BASE_IMAGE_VERSION}
16

17 18 19
  CMAKE_FLAGS:
    -DDUNE_PYTHON_VIRTUALENV_SETUP=True
    -DDUNE_PYTHON_ALLOW_GET_PIP=True
20 21
  # Recursively load submodules by default
  GIT_SUBMODULE_STRATEGY: recursive
22 23
  MAKE_FLAGS:
    -j $CPUS_MULTICORE
24
  RUN_IN_DUNE_ENV: $CI_PROJECT_DIR/build-cmake/run-in-dune-env
25

26 27 28 29 30 31
  # Documentation server configuration
  DOC_HOST: root@hermes.iup.uni-heidelberg.de
  DOC_PORT: 2345      # ... forwards to utopia_doc_server container. root ok ;)
  DOC_REMOTE_BASE_DIR: /var/dorie_doc
  DOC_REMOTE_PATH: $DOC_REMOTE_BASE_DIR/$CI_COMMIT_REF_SLUG

32
image: $DUNE_ENV_IMAGE
Dion Haefner's avatar
Dion Haefner committed
33

34 35 36 37
# Run pipelines on branches
include:
  - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'

38
# Move into top-level DUNE directory
Dion Haefner's avatar
Dion Haefner committed
39
before_script:
40
  - cd $DUNE_DIR
Dion Haefner's avatar
Dion Haefner committed
41

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
# Provide SSH access via the SSH_PRIVATE_KEY and SSH_KNOWN_HOSTS variables
.ssh-access: &ssh_access
  before_script:
    # Run ssh-agent (inside the build environment)
    - eval $(ssh-agent -s)

    # Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
    # We're using tr to fix line endings which makes ed25519 keys work
    # without extra base64 encoding.
    # https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null

    # Create the SSH directory and give it the right permissions
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh

    # Add the known hosts lists to ensure this ssh connection is the right one
    - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts


Dion Haefner's avatar
Dion Haefner committed
63
stages:
64
  - setup
Lukas Riedel's avatar
Lukas Riedel committed
65
  - prep
66
  - build
67
  - test
68 69
  - deploy

70 71
# --- Job Template for Docker in Docker (DinD) Runner Setup ---
.dind:
72 73 74 75 76 77 78
  tags:
    - dind
  image: docker:stable
  services:
    - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
79
  before_script:
80
    - $DOCKER_LOGIN
81 82 83 84 85 86 87 88

# --- Preparation of Docker Images ---
setup:dune-env: &setup
  stage: setup
  extends: .dind
  rules:
    - changes:
        - docker/dune-env.dockerfile
89
  script:
90
    - docker build -f docker/dune-env.dockerfile
91
      --build-arg PROCNUM=$CPUS_DIND
92
      -t $DUNE_ENV_IMAGE .
93
    - docker push $DUNE_ENV_IMAGE
94 95 96 97 98 99

setup:dune-env-clang:
  <<: *setup
  script:
    - docker build -f docker/dune-env.dockerfile
      --build-arg CC=clang
100
      --build-arg CXX=clang++
101 102 103
      --build-arg PROCNUM=$CPUS_DIND
      -t ${DUNE_ENV_IMAGE}-clang .
    - docker push ${DUNE_ENV_IMAGE}-clang
Dion Haefner's avatar
Dion Haefner committed
104

105
prep:update-dune: &update
106
  stage: prep
107
  extends: .dind
Lukas Riedel's avatar
Lukas Riedel committed
108
  allow_failure: true
109 110 111 112 113 114 115 116
  rules:
    # Run after pushes to master and tags in original repo, not forks
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "dorie/dorie"'
    - if: '$CI_COMMIT_TAG && $CI_PROJECT_PATH == "dorie/dorie"'
    # Run after Dockerfile changed. Do not allow failure then.
    - changes:
        - docker/dune-env-update.dockerfile
      allow_failure: false
117
  script:
118
    - docker build -f docker/dune-env-update.dockerfile
119 120
      --build-arg DUNE_ENV_IMAGE=$DUNE_ENV_IMAGE
      --build-arg PROCNUM=$CPUS_DIND
121
      -t $DUNE_ENV_IMAGE .
122
    - docker push $DUNE_ENV_IMAGE
123

124 125 126 127
prep:update-dune-clang:
  <<: *update
  script:
    - docker build -f docker/dune-env-update.dockerfile
128
      --build-arg DUNE_ENV_IMAGE=${DUNE_ENV_IMAGE}-clang
129
      --build-arg PROCNUM=$CPUS_DIND
130 131
      -t ${DUNE_ENV_IMAGE}-clang .
    - docker push ${DUNE_ENV_IMAGE}-clang
132 133 134


# --- Build jobs ---
135
build:system-tests: &build-tests
136
  stage: build
137 138
  tags:
    - multicore
139 140 141
  # Reduce concurrent jobs due to RAM limitations
  variables:
    MAKE_FLAGS: -j 2
Dion Haefner's avatar
Dion Haefner committed
142
  script:
143
    - CMAKE_FLAGS="$CMAKE_FLAGS -DCOVERAGE_REPORT=On"
144
      $DUNECONTROL --only=dorie configure
145
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS dorie-rfg
146
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_system_tests
147
  artifacts:
148
    name: "$CI_JOB_NAME"
149
    paths:
150
      - $CI_PROJECT_DIR/build-cmake
Lukas Riedel's avatar
Lukas Riedel committed
151
    expire_in: 1 day
152

153 154 155
build:unit-tests:
  <<: *build-tests
  script:
156 157
    - CMAKE_FLAGS="$CMAKE_FLAGS -DCOVERAGE_REPORT=On
                   -DCMAKE_BUILD_TYPE=None"
158
      $DUNECONTROL --only=dorie configure
159
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_unit_tests
160

161
build:debug: &debug
162
  stage: build
163 164
  tags:
    - multicore
165
  allow_failure: true
166 167
  before_script:
    - cd $DUNE_DIR
Lukas Riedel's avatar
Lukas Riedel committed
168 169
    - CMAKE_FLAGS="
        $CMAKE_FLAGS
170
        -DCMAKE_BUILD_TYPE=Debug
Lukas Riedel's avatar
Lukas Riedel committed
171
        -DCMAKE_CXX_FLAGS_DEBUG='-Werror'"
172
      $DUNECONTROL --only=dorie configure
173
  script:
174
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS dorie-rfg
175
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_unit_tests
176
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS richards_d2_r1 transport_d2_r0_t0
177

178 179 180
build:debug-clang:
  <<: *debug
  image: ${DUNE_ENV_IMAGE}-clang
181 182 183
  before_script:
    - cd $DUNE_DIR
    # Choose Clang compiler explicitly
184 185
    - CMAKE_FLAGS="
        $CMAKE_FLAGS
186
        -DCMAKE_BUILD_TYPE=Debug
187 188 189 190
        -DCMAKE_C_COMPILER=clang
        -DCMAKE_CXX_COMPILER=clang++
        -DCMAKE_CXX_FLAGS_DEBUG='-Werror'"
      $DUNECONTROL --only=dorie configure
191

192 193 194 195 196 197 198 199 200 201 202
build:docs:
  stage: build
  script:
    - CMAKE_FLAGS="$CMAKE_FLAGS"
      $DUNECONTROL --only=dorie configure
    - $DUNECONTROL --only=dorie make doc
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
      - $CI_PROJECT_DIR/build-cmake/doc
    expire_in: 1 day
203 204

# --- Tests ---
205
test:system-tests: &test
Lukas Riedel's avatar
Lukas Riedel committed
206 207
  stage: test
  dependencies:
208
    - build:system-tests
209 210 211 212 213 214
  needs: ["build:system-tests"]
  variables:
    GCOVR_PATHS:
      --root $CI_PROJECT_DIR
      --exclude $CI_PROJECT_DIR/dune/dorie/test
      --exclude $CI_PROJECT_DIR/plugins
Lukas Riedel's avatar
Lukas Riedel committed
215
  script:
Lukas Riedel's avatar
Lukas Riedel committed
216
    - $DUNECONTROL --only=dorie configure
217
    - $DUNECONTROL --only=dorie make system_tests
218 219 220 221 222 223 224 225 226 227
  # Run coverage utility (will also run after failed job)
  after_script:
    - source $CI_PROJECT_DIR/build-cmake/activate
    - pip3 install gcovr
    - cd $CI_PROJECT_DIR/build-cmake
    - mkdir coverage
    - gcovr $GCOVR_PATHS --html --html-details -o coverage/report.html
    # run again for coverage report in GitLab
    - gcovr $GCOVR_PATHS
  coverage: '/^TOTAL.*\s+(\d+\%)$/'
Lukas Riedel's avatar
Lukas Riedel committed
228 229 230
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
231
      - $CI_PROJECT_DIR/build-cmake/test
232
      - $CI_PROJECT_DIR/build-cmake/coverage
233
    expire_in: 1 day
Lukas Riedel's avatar
Lukas Riedel committed
234

235
test:unit-tests:
236
  <<: *test
237 238
  dependencies:
    - build:unit-tests
239
  needs: ["build:unit-tests"]
240 241
  script:
    - $DUNECONTROL --only=dorie configure
242
    - $DUNECONTROL --only=dorie make unit_tests
243 244 245 246
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
      - $CI_PROJECT_DIR/build-cmake/dune/dorie/test
247
      - $CI_PROJECT_DIR/build-cmake/coverage
248
    expire_in: 1 day
249

250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
test:examples:
  stage: test
  dependencies:
    - build:system-tests
  needs: ["build:system-tests"]
  only:
    - schedules
    - master@dorie/dorie
  script:
    - $DUNECONTROL --only=dorie configure
    - $DUNECONTROL --only=dorie make example_tests
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
      - $CI_PROJECT_DIR/build-cmake/doc
    expire_in: 1 day

267 268
test:python-tests:
  <<: *test
269
  # May run immediately
270
  dependencies: []
271
  needs: []
272 273 274 275
  script:
    - $DUNECONTROL --only=dorie configure
    - $DUNECONTROL --only=dorie make test_python

276
# --- Deploy jobs ---
277
deploy:dockerhub-devel: &deploy
Lukas Riedel's avatar
Lukas Riedel committed
278
  stage: deploy
279 280 281 282
  extends: .dind
  rules:
    # Run after pushes to master in original repo, not forks
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "dorie/dorie"'
283
  script:
284
    - docker build -f docker/dorie.dockerfile
285
      --build-arg BASE_IMG_VERSION=$IMAGE_VERSION --build-arg PROCNUM=$CPUS_DIND
286
      -t dorie/dorie:devel .
287
    - docker push dorie/dorie:devel
288 289 290
  environment:
    name: docker/devel
    url: https://hub.docker.com/r/dorie/dorie
291 292

deploy:dockerhub-stable:
293
  <<: *deploy
294 295 296
  rules:
    # Run after pushes to tags in original repo, not forks
    - if: '$CI_COMMIT_TAG && $CI_PROJECT_PATH == "dorie/dorie"'
297
  script:
298
    - docker build -f docker/dorie.dockerfile
299
      --build-arg BASE_IMG_VERSION=$IMAGE_VERSION --build-arg PROCNUM=$CPUS_DIND
300
      -t dorie/dorie:$CI_COMMIT_TAG .
301
    - docker push dorie/dorie:$CI_COMMIT_TAG
302 303 304
  environment:
    name: docker/$CI_COMMIT_TAG
    url: https://hub.docker.com/r/dorie/dorie
305

306
deploy:docs:
307
  stage: deploy
308
  only:
309
    - branches@dorie/dorie
310
  dependencies:
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
    - build:docs
  needs: ["build:docs"]

  <<: *ssh_access
  script:
    # Create the directory on the remote, removing any prior version
    - echo "Creating remote directory $DOC_REMOTE_PATH ..."
    - ssh -p $DOC_PORT $DOC_HOST "rm -rf $DOC_REMOTE_PATH"
    - ssh -p $DOC_PORT $DOC_HOST "mkdir -p $DOC_REMOTE_PATH/doxygen/"

    # Copy sphinx & doxygen HTML documentation to remote
    - cd build-cmake/doc
    - echo "Uploading documentation to $DOC_REMOTE_PATH/ ..."
    - scp -P $DOC_PORT -pr html         $DOC_HOST:$DOC_REMOTE_PATH/
    - scp -P $DOC_PORT -pr doxygen/html $DOC_HOST:$DOC_REMOTE_PATH/doxygen/

  environment:
    name: docs/$CI_COMMIT_REF_NAME
    url: https://hermes.iup.uni-heidelberg.de/dorie_doc/$CI_COMMIT_REF_SLUG/html/
    on_stop: deploy:stop_docs

# This job is called when the environment is stopped, which automatically
# happens when the respective branch is deleted
deploy:stop_docs:
  stage: deploy
  when: manual
  variables:
    # Stop GitLab from checking out the commit again (branch is deleted)
    GIT_STRATEGY: none
  dependencies: []
  needs: ["build:docs"]

  <<: *ssh_access
  script:
    - echo "Removing remote directory $DOC_REMOTE_PATH ..."
    - ssh -p $DOC_PORT $DOC_HOST "rm -rf $DOC_REMOTE_PATH"

348
  environment:
349 350
    name: docs/$CI_COMMIT_REF_NAME
    action: stop