difftreelog
set timeout for logs and disable second logs report
in: master
(cherry picked from commit d5b272411f47a0075badb1b708909c3cd823bd98)
1 file changed
.github/workflows/node-only-update.ymldiffbeforeafterboth1# https://cryptousetech.atlassian.net/wiki/spaces/CI/pages/2586837028/Nodes+only+update2# Node only update with restart polkadot-launch process.34name: nodes-only update56# Triger: only call from main workflow(re-usable workflows)7on:8 workflow_call:91011# A workflow run is made up of one or more jobs that can run sequentially or in parallel12jobs:1314 nodes-execution-matrix:1516 name: execution matrix1718 runs-on: self-hosted-ci19 outputs:20 matrix: ${{ steps.create_matrix.outputs.matrix }}2122 steps:2324 - name: Clean Workspace25 uses: AutoModality/action-clean@v1.1.02627 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it28 - uses: actions/checkout@v329 with:30 ref: ${{ github.head_ref }} #Checking out head commit3132 - name: Read .env file33 uses: xom9ikk/dotenv@v23435 - name: Create Execution matrix36 uses: CertainLach/create-matrix-action@v337 id: create_matrix38 with:39 matrix: |40 network {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}41 network {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}42 network {unique}, mainnet_branch {${{ env.UNIQUE_MAINNET_BRANCH }}}43444546 nodes-only-update:47 needs: nodes-execution-matrix48 # The type of runner that the job will run on49 runs-on: [self-hosted-ci,large]5051 timeout-minutes: 2880 # 48 hours for execution jobs.5253 name: ${{ matrix.network }}5455 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.5657 strategy:58 matrix:59 include: ${{fromJson(needs.nodes-execution-matrix.outputs.matrix)}}6061 steps:6263 - name: Clean Workspace64 uses: AutoModality/action-clean@v1.1.06566 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it67 - uses: actions/checkout@v368 with:69 ref: ${{ github.head_ref }} #Checking out head commit7071 - name: Read .env file72 uses: xom9ikk/dotenv@v27374 - name: Generate ENV related extend file for docker-compose75 uses: cuchi/jinja2-action@v1.2.076 with:77 template: .docker/docker-compose.tmp-node.j278 output_file: .docker/docker-compose.node.${{ matrix.network }}.yml79 variables: |80 REPO_URL=${{ github.server_url }}/${{ github.repository }}.git81 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}82 POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }}83 POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }}84 POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }}85 MAINNET_TAG=${{ matrix.mainnet_tag }}86 MAINNET_BRANCH=${{ matrix.mainnet_branch }}87 NETWORK=${{ matrix.network }}88 BRANCH=${{ github.head_ref }}8990 - name: Show build configuration91 run: cat .docker/docker-compose.node.${{ matrix.network }}.yml9293 - name: Generate launch-config-forkless-nodata.json94 uses: cuchi/jinja2-action@v1.2.095 with:96 template: .docker/forkless-config/launch-config-forkless-nodata.j297 output_file: .docker/launch-config-forkless-nodata.json98 variables: |99 NETWORK=${{ matrix.network }}100101 - name: Show launch-config-forkless configuration102 run: cat .docker/launch-config-forkless-nodata.json103104 - uses: actions/setup-node@v3105 with:106 node-version: 16107108 - name: Build the stack109 run: docker-compose -f ".docker/docker-compose.node.${{ matrix.network }}.yml" up -d --build --remove-orphans --force-recreate --timeout 300110111 # 🚀 POLKADOT LAUNCH COMPLETE 🚀112 - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade.113 if: success()114 run: |115 counter=160116 function check_container_status {117 docker inspect -f {{.State.Running}} node-parachain118 }119 function do_docker_logs {120 docker logs --details node-parachain 2>&1121 }122 function is_started {123 if [ "$(check_container_status)" == "true" ]; then124 echo "Container: node-parachain RUNNING";125 echo "Check Docker logs"126 DOCKER_LOGS=$(do_docker_logs)127 if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then128 echo "🚀 POLKADOT LAUNCH COMPLETE 🚀"129 return 0130 else131 echo "Message not found in logs output, repeating..."132 return 1133 fi134 else135 echo "Container node-parachain NOT RUNNING"136 echo "Halting all future checks"137 exit 1138 fi139 echo "something goes wrong"140 exit 1141 }142 while ! is_started; do143 echo "Waiting for special message in log files "144 sleep 30s145 counter=$(( $counter - 1 ))146 echo "Counter: $counter"147 if [ "$counter" -gt "0" ]; then148 continue149 else150 break151 fi152 done153 echo "Halting script"154 exit 0155 shell: bash156157 - name: Checkout at '${{ matrix.mainnet_branch }}' branch158 uses: actions/checkout@master159 with:160 ref: ${{ matrix.mainnet_branch }} #Checking out head commit161 path: ${{ matrix.mainnet_branch }}162163 - name: Run tests before Node Parachain upgrade164 working-directory: ${{ matrix.mainnet_branch }}/tests165 run: |166 yarn install167 yarn add mochawesome168 node scripts/readyness.js169 echo "Ready to start tests"170 yarn polkadot-types171 NOW=$(date +%s) && yarn testConnection --reporter mochawesome --reporter-options reportFilename=test-before-${NOW}172 env:173 RPC_URL: http://127.0.0.1:9933/174175 - name: Upload Test Report Before Node upgrade176 uses: phoenix-actions/test-reporting@v8177 id: test-report-before178 if: success() || failure() # run this step even if previous step failed179 with:180 name: Tests before node upgrade ${{ matrix.network }} # Name of the check run which will be created181 path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-before-*.json # Path to test results182 reporter: mochawesome-json183 fail-on-error: 'false'184 token: ${{ secrets.GITHUB_TOKEN }}185186 # TODO uncomment thease steps after the merge187 #- name: Run Parallel tests before Node Parachain upgrade188 # working-directory: ${{ matrix.mainnet_branch }}/tests189 # run: |190 # yarn install191 # yarn add mochawesome192 # echo "Ready to start tests"193 # yarn polkadot-types194 # NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW}195 # env:196 # RPC_URL: http://127.0.0.1:9933/197198 #- name: Upload Parallel Test Report Before Node upgrade199 # uses: phoenix-actions/test-reporting@v8200 # id: test-parallel-report-before201 # if: success() || failure() # run this step even if previous step failed202 # with:203 # name: Tests before node upgrade ${{ matrix.network }} # Name of the check run which will be created204 # path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-parallel-*.json # Path to test results205 # reporter: mochawesome-json206 # fail-on-error: 'false'207208 # - name: Run Sequential tests before Node Parachain upgrade209 # if: success() || failure()210 # working-directory: ${{ matrix.mainnet_branch }}/tests211 # run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW}212 # env:213 # RPC_URL: http://127.0.0.1:9933/214215 # - name: Upload Sequential Test Report Before Node upgrade216 # uses: phoenix-actions/test-reporting@v8217 # id: test-sequential-report-before218 # if: success() || failure() # run this step even if previous step failed219 # with:220 # name: Tests before node upgrade ${{ matrix.network }} # Name of the check run which will be created221 # path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-sequential-*.json # Path to test results222 # reporter: mochawesome-json223 # fail-on-error: 'false'224225 - name: Send SIGUSR1 to polkadot-launch process226 if: success() || failure()227 run: |228 #Get PID of polkadot-launch229 ContainerID=$(docker ps -aqf "name=node-parachain")230 PID=$(docker exec node-parachain pidof 'polkadot-launch')231 sleep 30s232 echo -e "Show logs of node-parachain container.\n"233 docker logs ${ContainerID}234 echo -e "\n"235 echo -e "Restart polkadot-launch process: $PID\n"236 docker exec node-parachain kill -SIGUSR1 ${PID}237 echo "SIGUSR1 sent to Polkadot-launch PID: $PID"238 docker logs ${ContainerID}239240 - name: Get chain logs in case of docker image crashed after Polkadot Launch restart241 if: failure() # run this step only at failure242 run: |243 docker exec node-parachain tail -n 1000 /polkadot-launch/9944.log244 docker exec node-parachain tail -n 1000 /polkadot-launch/9945.log245 docker exec node-parachain tail -n 1000 /polkadot-launch/alice.log246247 - name: copy chain log files from container to the host248 if: success() || failure() # run this step even if previous step failed249 run: |250 mkdir -p /tmp/node-only-update251 docker cp node-parachain:/polkadot-launch/9944.log /tmp/node-only-update/252 docker cp node-parachain:/polkadot-launch/9945.log /tmp/node-only-update/253 docker cp node-parachain:/polkadot-launch/alice.log /tmp/node-only-update/254255 - name: Upload chain log files256 if: success() || failure()257 uses: actions/upload-artifact@v3258 with:259 name: node-only-update-chain-logs260 path: /tmp/node-only-update/261 if-no-files-found: warn262263 - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade.264 if: success()265 run: |266 counter=160267 function check_container_status {268 docker inspect -f {{.State.Running}} node-parachain269 }270 function do_docker_logs {271 docker logs --details node-parachain 2>&1272 }273 function is_started {274 if [ "$(check_container_status)" == "true" ]; then275 echo "Container: node-parachain RUNNING";276 echo "Check Docker logs"277 DOCKER_LOGS=$(do_docker_logs)278 if [[ ${DOCKER_LOGS} = *"All parachain collators restarted with the new binaries."* ]];then279 echo "🌗 All parachain collators restarted with the new binaries."280 return 0281 else282 echo "Message not found in logs output, repeating..."283 return 1284 fi285 else286 echo "Container node-parachain NOT RUNNING"287 echo "Halting all future checks"288 exit 1289 fi290 echo "something goes wrong"291 exit 1292 }293 while ! is_started; do294 echo "Waiting for special message in log files "295 sleep 30s296 counter=$(( $counter - 1 ))297 echo "Counter: $counter"298 if [ "$counter" -gt "0" ]; then299 continue300 else301 break302 fi303 done304 echo "Halting script"305 exit 0306 shell: bash307308 ## TODO: Remove next two blocks before switch to Parrallel & Sequental tests. Uncoment commented blocks.309 - name: Run tests after Node Parachain upgrade310 if: success() || failure() # run this step even if previous step failed311 working-directory: ${{ matrix.mainnet_branch }}/tests312 run: |313 yarn install314 yarn add mochawesome315 node scripts/readyness.js316 echo "Ready to start tests"317 yarn polkadot-types318 NOW=$(date +%s) && yarn test --reporter mochawesome --reporter-options reportFilename=test-after-${NOW}319 env:320 RPC_URL: http://127.0.0.1:9933/321322 - name: Test Report After Node upgrade323 uses: phoenix-actions/test-reporting@v8324 id: test-report-after325 if: success() || failure() # run this step even if previous step failed326 with:327 name: Tests after node upgrade ${{ matrix.network }} # Name of the check run which will be created328 path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-after-*.json # Path to test results329 reporter: mochawesome-json330 fail-on-error: 'false'331 token: ${{ secrets.GITHUB_TOKEN }}332333 # TODO uncomment thease steps after the merge334 #- name: Run Parallel tests after Node Parachain upgrade335 # working-directory: ${{ matrix.mainnet_branch }}/tests336 # run: |337 # yarn install338 # yarn add mochawesome339 # node scripts/readyness.js340 # echo "Ready to start tests"341 # yarn polkadot-types342 # NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW}343 # env:344 # RPC_URL: http://127.0.0.1:9933/345346 #- name: Test Report Parallel After Node upgrade347 # uses: phoenix-actions/test-reporting@v8348 # id: test-report-parallel-after349 # if: success() || failure() # run this step even if previous step failed350 # with:351 # name: Tests after node upgrade ${{ matrix.network }} # Name of the check run which will be created352 # path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-parallel-*.json # Path to test results353 # reporter: mochawesome-json354 # fail-on-error: 'false'355356 #- name: Run Sequential tests after Node Parachain upgrade357 # if: success() || failure()358 # working-directory: ${{ matrix.mainnet_branch }}/tests359 # run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW}360 # env:361 # RPC_URL: http://127.0.0.1:9933/362363 #- name: Upload Sequential Test Report After Node upgrade364 # uses: phoenix-actions/test-reporting@v8365 # id: test-sequential-report-after366 # if: success() || failure() # run this step even if previous step failed367 # with:368 # name: Tests before node upgrade ${{ matrix.network }} # Name of the check run which will be created369 # path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-sequential-*.json # Path to test results370 # reporter: mochawesome-json371 # fail-on-error: 'false'372373374 - name: Stop running containers375 if: always() # run this step always376 run: docker-compose -f ".docker/docker-compose.node.${{ matrix.network }}.yml" down --volumes377378 - name: Remove builder cache379 if: always() # run this step always380 run: |381 docker builder prune -f -a382 docker system prune -f383 docker image prune -f -a384385 - name: Remove repo at the end386 if: always() # run this step always387 run: |388 ls -ls ./