.gitlab-ci.yml 8.97 KB
Newer Older
Dion Haefner's avatar
Dion Haefner committed
1
variables:
2 3
  DUNE_DIR: /opt/dune
  DUNE_CONTROL_PATH: $DUNE_DIR:$CI_PROJECT_DIR
4
  DUNECONTROL: ./dune-common/bin/dunecontrol
Lukas Riedel's avatar
Lukas Riedel committed
5
  CPUS_MULTICORE: 4
6
  CPUS_DIND: 2
7 8
  DOCKER_LOGIN:
    docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PW
9 10 11 12 13
  
  # DUNE environment image
  BASE_IMAGE: dorie/dune-env
  # Use semantic versioning (not the version of DUNE) and bump according to
  # to whether changes are backwards-compatible or not.
14
  IMAGE_VERSION: "1.3"
15 16
  DUNE_ENV_IMAGE: ${BASE_IMAGE}:img-v${IMAGE_VERSION}

17 18 19
  CMAKE_FLAGS:
    -DDUNE_PYTHON_VIRTUALENV_SETUP=True
    -DDUNE_PYTHON_ALLOW_GET_PIP=True
20 21
  # Recursively load submodules by default
  GIT_SUBMODULE_STRATEGY: recursive
22 23
  MAKE_FLAGS:
    -j $CPUS_MULTICORE
24
  RUN_IN_DUNE_ENV: $CI_PROJECT_DIR/build-cmake/run-in-dune-env
25

26 27 28 29 30 31
  # Documentation server configuration
  DOC_HOST: root@hermes.iup.uni-heidelberg.de
  DOC_PORT: 2345      # ... forwards to utopia_doc_server container. root ok ;)
  DOC_REMOTE_BASE_DIR: /var/dorie_doc
  DOC_REMOTE_PATH: $DOC_REMOTE_BASE_DIR/$CI_COMMIT_REF_SLUG

32
image: $DUNE_ENV_IMAGE
Dion Haefner's avatar
Dion Haefner committed
33

34
# Move into top-level DUNE directory
Dion Haefner's avatar
Dion Haefner committed
35
before_script:
36
  - cd $DUNE_DIR
Dion Haefner's avatar
Dion Haefner committed
37

38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
# Provide SSH access via the SSH_PRIVATE_KEY and SSH_KNOWN_HOSTS variables
.ssh-access: &ssh_access
  before_script:
    # Run ssh-agent (inside the build environment)
    - eval $(ssh-agent -s)

    # Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
    # We're using tr to fix line endings which makes ed25519 keys work
    # without extra base64 encoding.
    # https://gitlab.com/gitlab-examples/ssh-private-key/issues/1#note_48526556
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null

    # Create the SSH directory and give it the right permissions
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh

    # Add the known hosts lists to ensure this ssh connection is the right one
    - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts


Dion Haefner's avatar
Dion Haefner committed
59
stages:
60
  - setup
Lukas Riedel's avatar
Lukas Riedel committed
61
  - prep
62
  - build
63
  - test
64 65
  - deploy

66 67

# --- Preparation of Docker Images ---
68
setup:dune-env: &setup
69
  stage: setup
70
  only:
71 72
    variables:
      - $REBUILD_BASE_IMAGE # must be defined and have non-empty value
73 74 75 76 77 78 79
  tags:
    - dind
  image: docker:stable
  services:
    - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
80
  before_script:
81
    - $DOCKER_LOGIN
82
  script:
83
    - docker build -f docker/dune-env.dockerfile
84
      --build-arg PROCNUM=$CPUS_DIND
85
      -t $DUNE_ENV_IMAGE .
86
    - docker push $DUNE_ENV_IMAGE
87 88 89 90 91 92

setup:dune-env-clang:
  <<: *setup
  script:
    - docker build -f docker/dune-env.dockerfile
      --build-arg CC=clang
93
      --build-arg CXX=clang++
94 95 96
      --build-arg PROCNUM=$CPUS_DIND
      -t ${DUNE_ENV_IMAGE}-clang .
    - docker push ${DUNE_ENV_IMAGE}-clang
Dion Haefner's avatar
Dion Haefner committed
97

98
prep:update-dune: &update
99
  <<: *setup
100
  stage: prep
101
  allow_failure: true
102 103 104
  only:
    - master
    - tags
105
  script:
106
    - docker build -f docker/dune-env-update.dockerfile
107 108
      --build-arg DUNE_ENV_IMAGE=$DUNE_ENV_IMAGE
      --build-arg PROCNUM=$CPUS_DIND
109
      -t $DUNE_ENV_IMAGE .
110
    - docker push $DUNE_ENV_IMAGE
111

112 113 114 115
prep:update-dune-clang:
  <<: *update
  script:
    - docker build -f docker/dune-env-update.dockerfile
116
      --build-arg DUNE_ENV_IMAGE=${DUNE_ENV_IMAGE}-clang
117 118
      --build-arg CC=clang
      --build-arg CXX=clang++
119
      --build-arg PROCNUM=$CPUS_DIND
120 121
      -t ${DUNE_ENV_IMAGE}-clang .
    - docker push ${DUNE_ENV_IMAGE}-clang
122 123 124


# --- Build jobs ---
125
build:system-tests: &build-tests
126
  stage: build
127 128 129
  # Reduce concurrent jobs due to RAM limitations
  variables:
    MAKE_FLAGS: -j 2
Dion Haefner's avatar
Dion Haefner committed
130
  script:
131
    - CMAKE_FLAGS="$CMAKE_FLAGS -DCOVERAGE_REPORT=On"
132
      $DUNECONTROL --only=dorie configure
133
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS dorie-rfg
134
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_system_tests
135
  artifacts:
136
    name: "$CI_JOB_NAME"
137
    paths:
138
      - $CI_PROJECT_DIR/build-cmake
Lukas Riedel's avatar
Lukas Riedel committed
139
    expire_in: 1 day
140

141 142 143
build:unit-tests:
  <<: *build-tests
  script:
144 145
    - CMAKE_FLAGS="$CMAKE_FLAGS -DCOVERAGE_REPORT=On
                   -DCMAKE_BUILD_TYPE=None"
146
      $DUNECONTROL --only=dorie configure
147
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_unit_tests
148

149
build:debug: &debug
150
  stage: build
151
  allow_failure: true
152 153
  before_script:
    - cd $DUNE_DIR
Lukas Riedel's avatar
Lukas Riedel committed
154 155
    - CMAKE_FLAGS="
        $CMAKE_FLAGS
156
        -DCMAKE_BUILD_TYPE=Debug
Lukas Riedel's avatar
Lukas Riedel committed
157
        -DCMAKE_CXX_FLAGS_DEBUG='-Werror'"
158
      $DUNECONTROL --only=dorie configure
159
  script:
160
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS dorie-rfg
161
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS build_unit_tests
162
    - $DUNECONTROL --only=dorie make $MAKE_FLAGS richards_d2_r1 transport_d2_r0_t0
163

164 165 166
build:debug-clang:
  <<: *debug
  image: ${DUNE_ENV_IMAGE}-clang
167 168 169
  before_script:
    - cd $DUNE_DIR
    # Choose Clang compiler explicitly
170 171
    - CMAKE_FLAGS="
        $CMAKE_FLAGS
172
        -DCMAKE_BUILD_TYPE=Debug
173 174 175 176
        -DCMAKE_C_COMPILER=clang
        -DCMAKE_CXX_COMPILER=clang++
        -DCMAKE_CXX_FLAGS_DEBUG='-Werror'"
      $DUNECONTROL --only=dorie configure
177

178 179 180 181 182 183 184 185 186 187 188
build:docs:
  stage: build
  script:
    - CMAKE_FLAGS="$CMAKE_FLAGS"
      $DUNECONTROL --only=dorie configure
    - $DUNECONTROL --only=dorie make doc
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
      - $CI_PROJECT_DIR/build-cmake/doc
    expire_in: 1 day
189 190

# --- Tests ---
191
test:system-tests: &test
Lukas Riedel's avatar
Lukas Riedel committed
192 193
  stage: test
  dependencies:
194
    - build:system-tests
195 196 197 198 199 200
  needs: ["build:system-tests"]
  variables:
    GCOVR_PATHS:
      --root $CI_PROJECT_DIR
      --exclude $CI_PROJECT_DIR/dune/dorie/test
      --exclude $CI_PROJECT_DIR/plugins
Lukas Riedel's avatar
Lukas Riedel committed
201
  script:
Lukas Riedel's avatar
Lukas Riedel committed
202
    - $DUNECONTROL --only=dorie configure
203
    - $DUNECONTROL --only=dorie make system_tests
204 205 206 207 208 209 210 211 212 213
  # Run coverage utility (will also run after failed job)
  after_script:
    - source $CI_PROJECT_DIR/build-cmake/activate
    - pip3 install gcovr
    - cd $CI_PROJECT_DIR/build-cmake
    - mkdir coverage
    - gcovr $GCOVR_PATHS --html --html-details -o coverage/report.html
    # run again for coverage report in GitLab
    - gcovr $GCOVR_PATHS
  coverage: '/^TOTAL.*\s+(\d+\%)$/'
Lukas Riedel's avatar
Lukas Riedel committed
214 215 216
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
217
      - $CI_PROJECT_DIR/build-cmake/test
218
      - $CI_PROJECT_DIR/build-cmake/coverage
219
    expire_in: 1 day
Lukas Riedel's avatar
Lukas Riedel committed
220

221
test:unit-tests:
222
  <<: *test
223 224
  dependencies:
    - build:unit-tests
225
  needs: ["build:unit-tests"]
226 227
  script:
    - $DUNECONTROL --only=dorie configure
228
    - $DUNECONTROL --only=dorie make unit_tests
229 230 231 232
  artifacts:
    name: "$CI_JOB_NAME"
    paths:
      - $CI_PROJECT_DIR/build-cmake/dune/dorie/test
233
      - $CI_PROJECT_DIR/build-cmake/coverage
234
    expire_in: 1 day
235

236 237 238
test:python-tests:
  <<: *test
  dependencies: []
239 240 241
  # TODO: Make empty once https://gitlab.com/gitlab-org/gitlab/issues/30631
  #       is resolved
  needs: ["build:docs"] # Any quick job from previous stage
242 243 244 245
  script:
    - $DUNECONTROL --only=dorie configure
    - $DUNECONTROL --only=dorie make test_python

246
# --- Deploy jobs ---
247
deploy:dockerhub-devel: &deploy
Lukas Riedel's avatar
Lukas Riedel committed
248
  stage: deploy
249
  only:
250
    - master@dorie/dorie
251 252 253 254 255 256 257
  tags:
    - dind
  image: docker:stable
  services:
    - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
258
  dependencies: []
259
  before_script:
260
    - $DOCKER_LOGIN
261
  script:
262
    - docker build -f docker/dorie.dockerfile
263
      --build-arg BASE_IMG_VERSION=$IMAGE_VERSION --build-arg PROCNUM=$CPUS_DIND
264
      -t dorie/dorie:devel .
265
    - docker push dorie/dorie:devel
266 267 268
  environment:
    name: docker/devel
    url: https://hub.docker.com/r/dorie/dorie
269 270

deploy:dockerhub-stable:
271
  <<: *deploy
272
  only:
273
    - tags@dorie/dorie
274
  script:
275
    - docker build -f docker/dorie.dockerfile
276
      --build-arg BASE_IMG_VERSION=$IMAGE_VERSION --build-arg PROCNUM=$CPUS_DIND
277
      -t dorie/dorie:$CI_COMMIT_TAG .
278
    - docker push dorie/dorie:$CI_COMMIT_TAG
279 280 281
  environment:
    name: docker/$CI_COMMIT_TAG
    url: https://hub.docker.com/r/dorie/dorie
282

283
deploy:docs:
284
  stage: deploy
285
  only:
286
    - branches@dorie/dorie
287
  dependencies:
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
    - build:docs
  needs: ["build:docs"]

  <<: *ssh_access
  script:
    # Create the directory on the remote, removing any prior version
    - echo "Creating remote directory $DOC_REMOTE_PATH ..."
    - ssh -p $DOC_PORT $DOC_HOST "rm -rf $DOC_REMOTE_PATH"
    - ssh -p $DOC_PORT $DOC_HOST "mkdir -p $DOC_REMOTE_PATH/doxygen/"

    # Copy sphinx & doxygen HTML documentation to remote
    - cd build-cmake/doc
    - echo "Uploading documentation to $DOC_REMOTE_PATH/ ..."
    - scp -P $DOC_PORT -pr html         $DOC_HOST:$DOC_REMOTE_PATH/
    - scp -P $DOC_PORT -pr doxygen/html $DOC_HOST:$DOC_REMOTE_PATH/doxygen/

  environment:
    name: docs/$CI_COMMIT_REF_NAME
    url: https://hermes.iup.uni-heidelberg.de/dorie_doc/$CI_COMMIT_REF_SLUG/html/
    on_stop: deploy:stop_docs

# This job is called when the environment is stopped, which automatically
# happens when the respective branch is deleted
deploy:stop_docs:
  stage: deploy
  when: manual
  variables:
    # Stop GitLab from checking out the commit again (branch is deleted)
    GIT_STRATEGY: none
  dependencies: []
  needs: ["build:docs"]

  <<: *ssh_access
  script:
    - echo "Removing remote directory $DOC_REMOTE_PATH ..."
    - ssh -p $DOC_PORT $DOC_HOST "rm -rf $DOC_REMOTE_PATH"

325
  environment:
326 327
    name: docs/$CI_COMMIT_REF_NAME
    action: stop