git.delta.rocks / unique-network / refs/commits / e3879d3a9287

1# https://cryptousetech.atlassian.net/wiki/spaces/CI/pages/2586837028/Nodes+only+update2# Node only update with restart polkadot-launch process.34name: nodes-only update56# Triger: only call from main workflow(re-usable workflows)7on:8  workflow_call:91011# A workflow run is made up of one or more jobs that can run sequentially or in parallel12jobs:1314  nodes-execution-matrix:1516    name: execution matrix1718    runs-on: self-hosted-ci19    outputs:20      matrix: ${{ steps.create_matrix.outputs.matrix }}2122    steps:2324      - name: Clean Workspace25        uses: AutoModality/action-clean@v1.1.02627      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it28      - uses: actions/checkout@v3.1.029        with:30          ref: ${{ github.head_ref }}  #Checking out head commit3132      - name: Read .env file33        uses: xom9ikk/dotenv@v23435      - name: Create Execution matrix36        uses: CertainLach/create-matrix-action@v437        id: create_matrix38        with:39          matrix: |40            network {opal}, wasm_name {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}41            network {sapphire}, wasm_name {quartz}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}42            network {quartz}, wasm_name {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}43            network {unique}, wasm_name {unique}, mainnet_branch {${{ env.UNIQUE_MAINNET_BRANCH }}}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}444546  nodes-only-update:47    needs: nodes-execution-matrix48    # The type of runner that the job will run on49    runs-on: [self-hosted-ci]5051    timeout-minutes: 2880           # 48 hours for execution jobs.5253    name: ${{ matrix.network }}5455    continue-on-error: false         #Do not stop testing of matrix runs failed.  As it decided during PR review - it required 50/50& Let's check it with false.5657    strategy:58      matrix:59        include: ${{fromJson(needs.nodes-execution-matrix.outputs.matrix)}}6061    steps:6263      - name: Clean Workspace64        uses: AutoModality/action-clean@v1.1.06566      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it67      - uses: actions/checkout@v3.1.068        with:69          ref: ${{ github.head_ref }}  #Checking out head commit7071      - name: Read .env file72        uses: xom9ikk/dotenv@v27374      - name: Generate ENV related extend file for docker-compose75        uses: cuchi/jinja2-action@v1.2.076        with:77          template: .docker/docker-compose.tmp-node.j278          output_file: .docker/docker-compose.node.${{ matrix.network }}.yml79          variables: |80            REPO_URL=${{ github.server_url }}/${{ github.repository }}.git81            RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}82            POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}83            POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }}84            # POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }}85            # MAINNET_TAG=${{ matrix.mainnet_tag }}86            MAINNET_BRANCH=${{ matrix.mainnet_branch }}87            NETWORK=${{ matrix.network }}88            WASM_NAME=${{ matrix.wasm_name }}89            # BRANCH=${{ github.head_ref }}9091      - name: Show build configuration92        run: cat .docker/docker-compose.node.${{ matrix.network }}.yml9394      - name: Generate launch-config-forkless-nodata.json95        uses: cuchi/jinja2-action@v1.2.096        with:97          template: .docker/forkless-config/launch-config-forkless-nodata.j298          output_file: .docker/launch-config-forkless-nodata.json99          variables: |100            NETWORK=${{ matrix.network }}101            WASM_NAME=${{ matrix.wasm_name }}102            RELAY_CHAIN_TYPE=${{ env.RELAY_CHAIN_TYPE }}103104      - name: Show launch-config-forkless configuration105        run: cat .docker/launch-config-forkless-nodata.json106107      - uses: actions/setup-node@v3.5.1108        with:109          node-version: 16110111      - name: Build the stack112        run: docker-compose -f ".docker/docker-compose.node.${{ matrix.network }}.yml" up -d --build --remove-orphans --force-recreate --timeout 300113114      #  🚀 POLKADOT LAUNCH COMPLETE 🚀115      - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade.116        if: success()117        run: |118          counter=160119          function check_container_status {120                docker inspect -f {{.State.Running}} node-parachain121          }122          function do_docker_logs {123                docker logs --details node-parachain  2>&1124          }125          function is_started {126                if [ "$(check_container_status)" == "true" ]; then127                        echo "Container: node-parachain RUNNING";128                        echo "Check Docker logs"129                        DOCKER_LOGS=$(do_docker_logs)130                        if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then131                                echo "🚀 POLKADOT LAUNCH COMPLETE 🚀"132                                return 0133                        else134                                echo "Message not found in logs output, repeating..."135                                return 1136                        fi137                else138                        echo "Container node-parachain NOT RUNNING"139                        echo "Halting all future checks"140                        exit 1141                fi142          echo "something goes wrong"143          exit 1144          }145          while ! is_started; do146                echo "Waiting for special message in log files "147                sleep 30s148                counter=$(( $counter - 1 ))149                echo "Counter: $counter"150                if [ "$counter" -gt "0" ]; then151                         continue152                else153                         break154                fi155          done156          echo "Halting script"157          exit 0158        shell: bash159160      - name: Checkout at '${{ matrix.mainnet_branch }}' branch161        uses: actions/checkout@master162        with:163          ref: ${{ matrix.mainnet_branch }}  #Checking out head commit164          path: ${{ matrix.mainnet_branch }}165166      - name: Run Parallel tests before Node Parachain upgrade167        working-directory: ${{ matrix.mainnet_branch }}/tests168        if: success() || failure()169        run: |170          yarn install171          yarn add mochawesome172          ./scripts/wait_for_first_block.sh173          echo "Ready to start tests"174          yarn polkadot-types175          NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW}176        env:177          RPC_URL: http://127.0.0.1:9933/178179      # - name: Upload Parallel Test Report Before Node upgrade180      #   uses: phoenix-actions/test-reporting@v8181      #   id: test-parallel-report-before182      #   if: success() || failure()    # run this step even if previous step failed183      #   with:184      #     name: Tests before node upgrade ${{ matrix.network }}            # Name of the check run which will be created185      #     path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-parallel-*.json    # Path to test results186      #     reporter: mochawesome-json187      #     fail-on-error: 'false'188189      - name: Run Sequential tests before Node Parachain upgrade190        if: success() || failure()191        working-directory: ${{ matrix.mainnet_branch }}/tests192        run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW}193        env:194          RPC_URL: http://127.0.0.1:9933/195196      # - name: Upload Sequential Test Report Before Node upgrade197      #   uses: phoenix-actions/test-reporting@v10198      #   id: test-sequential-report-before199      #   if: success() || failure()    # run this step even if previous step failed200      #   with:201      #     name: Tests before node upgrade ${{ matrix.network }}            # Name of the check run which will be created202      #     path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-sequential-*.json    # Path to test results203      #     reporter: mochawesome-json204      #     fail-on-error: 'false'205206      - name: Send SIGUSR1 to polkadot-launch process207        if: success() || failure()208        run: |209          #Get PID of polkadot-launch210          ContainerID=$(docker ps -aqf "name=node-parachain")211          PID=$(docker exec node-parachain pidof 'polkadot-launch')212          sleep 30s213          echo -e "\n"214          echo -e "Restart polkadot-launch process: $PID\n"215          docker exec node-parachain kill -SIGUSR1 ${PID}216          echo "SIGUSR1 sent to Polkadot-launch PID: $PID"217          sleep 60s218          echo -e "Show logs of node-parachain container.\n"219          docker logs ${ContainerID}220221      - name: Get chain logs in case of docker image crashed after Polkadot Launch restart222        if: failure()                   # run this step only at failure223        run: |224          docker exec node-parachain tail -n 1000 /polkadot-launch/9944.log225          docker exec node-parachain tail -n 1000 /polkadot-launch/9945.log226          docker exec node-parachain tail -n 1000 /polkadot-launch/alice.log227228      - name: copy chain log files from container to the host229        if: success() || failure()    # run this step even if previous step failed230        run: |231          mkdir -p /tmp/node-only-update232          docker cp node-parachain:/polkadot-launch/9944.log /tmp/node-only-update/233          docker cp node-parachain:/polkadot-launch/9945.log /tmp/node-only-update/234          docker cp node-parachain:/polkadot-launch/alice.log /tmp/node-only-update/235236      - name: Upload chain log files237        if: success() || failure()238        uses: actions/upload-artifact@v3239        with:240          name: node-only-update-chain-logs241          path: /tmp/node-only-update/242          if-no-files-found: warn243244      - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade.245        if: success()246        run: |247          counter=160248          function check_container_status {249                docker inspect -f {{.State.Running}} node-parachain250          }251          function do_docker_logs {252                docker logs --details node-parachain  2>&1253          }254          function is_started {255                if [ "$(check_container_status)" == "true" ]; then256                        echo "Container: node-parachain RUNNING";257                        echo "Check Docker logs"258                        DOCKER_LOGS=$(do_docker_logs)259                        if [[ ${DOCKER_LOGS} = *"All parachain collators restarted with the new binaries."* ]];then260                                echo "🌗 All parachain collators restarted with the new binaries."261                                return 0262                        else263                                echo "Message not found in logs output, repeating..."264                                return 1265                        fi266                else267                        echo "Container node-parachain NOT RUNNING"268                        echo "Halting all future checks"269                        exit 1270                fi271          echo "something goes wrong"272          exit 1273          }274          while ! is_started; do275                echo "Waiting for special message in log files "276                sleep 30s277                counter=$(( $counter - 1 ))278                echo "Counter: $counter"279                if [ "$counter" -gt "0" ]; then280                         continue281                else282                         break283                fi284          done285          echo "Halting script"286          exit 0287        shell: bash288289      - name: Run Parallel tests after Node Parachain upgrade290        working-directory: ${{ matrix.mainnet_branch }}/tests291        if: success() || failure()    # run this step even if previous step failed292        run: |293          yarn install294          yarn add mochawesome295          ./scripts/wait_for_first_block.sh296          echo "Ready to start tests"297          yarn polkadot-types298          NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW}299        env:300          RPC_URL: http://127.0.0.1:9933/301302      #- name: Test Report Parallel After Node upgrade303      #  uses: phoenix-actions/test-reporting@v10304      #  id: test-report-parallel-after305      #  if: success() || failure()    # run this step even if previous step failed306      #  with:307      #    name: Tests after node upgrade ${{ matrix.network }}            # Name of the check run which will be created308      #    path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-parallel-*.json    # Path to test results309      #    reporter: mochawesome-json310      #    fail-on-error: 'false'311312      - name: Run Sequential tests after Node Parachain upgrade313        if: success() || failure()314        working-directory: ${{ matrix.mainnet_branch }}/tests315        run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW}316        env:317          RPC_URL: http://127.0.0.1:9933/318319      #- name: Upload Sequential Test Report After Node upgrade320      #  uses: phoenix-actions/test-reporting@v10321      #  id: test-sequential-report-after322      #  if: success() || failure()    # run this step even if previous step failed323      #  with:324      #    name: Tests before node upgrade ${{ matrix.network }}            # Name of the check run which will be created325      #    path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-sequential-*.json    # Path to test results326      #    reporter: mochawesome-json327      #    fail-on-error: 'false'328329      - name: Stop running containers330        if: always()                   # run this step always331        run: docker-compose -f ".docker/docker-compose.node.${{ matrix.network }}.yml" down --volumes332333      - name: Remove builder cache334        if: always()                   # run this step always335        run: |336          docker builder prune -f -a337          docker system prune -f338          docker image prune -f -a339340      - name: Remove repo at the end341        if: always()                   # run this step always342        run: |343          ls -ls ./