git.delta.rocks / unique-network / refs/commits / 9acbee514302

1# https://cryptousetech.atlassian.net/wiki/spaces/CI/pages/2586837028/Nodes+only+update2# Node only update with restart polkadot-launch process.34name: nodes-only update56# Triger: only call from main workflow(re-usable workflows)7on:8  workflow_call:91011# A workflow run is made up of one or more jobs that can run sequentially or in parallel12jobs:1314  prepare-execution-matrix:1516    name: execution matrix1718    runs-on: self-hosted-ci19    outputs:20      matrix: ${{ steps.create_matrix.outputs.matrix }}2122    steps:2324      - name: Clean Workspace25        uses: AutoModality/action-clean@v1.1.02627      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it28      - uses: actions/checkout@v3.1.029        with:30          ref: ${{ github.head_ref }}  #Checking out head commit3132      - name: Read .env file33        uses: xom9ikk/dotenv@v23435      - name: Create Execution matrix36        uses: CertainLach/create-matrix-action@v437        id: create_matrix38        with:39          matrix: |40            network {opal}, wasm_name {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}41            network {sapphire}, wasm_name {quartz}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}42            network {quartz}, wasm_name {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}43            network {unique}, wasm_name {unique}, mainnet_branch {${{ env.UNIQUE_MAINNET_BRANCH }}}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}444546  node-only-update-build:4748    needs: prepare-execution-matrix49    # The type of runner that the job will run on50    runs-on: [self-hosted-ci]5152    timeout-minutes: 2880           # 48 hours for execution jobs.5354    name: ${{ matrix.network }}-build5556    continue-on-error: true         #Do not stop testing of matrix runs failed.  As it decided during PR review - it required 50/50& Let's check it with false.5758    strategy:59      matrix:60        include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}}6162    steps:6364      - name: Skip if pull request is in Draft65        if: github.event.pull_request.draft == true66        run: exit 16768      - name: Clean Workspace69        uses: AutoModality/action-clean@v1.1.07071      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it72      - uses: actions/checkout@v3.1.073        with:74          ref: ${{ github.head_ref }}  #Checking out head commit7576      # Prepare SHA  77      - name: Prepare SHA78        uses: ./.github/actions/prepare7980      - name: Read .env file81        uses: xom9ikk/dotenv@v28283      # Build main image for NODE-ONLY-UPDATE84      - name: Generate ENV related extend Dockerfile file85        uses: cuchi/jinja2-action@v1.2.086        with:87          template: .docker/Dockerfile-parachain-node-only.j288          output_file: .docker/Dockerfile-parachain-node-only.${{ matrix.network }}.yml89          variables: |90            RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}91            NETWORK=${{ matrix.network }}92            MAINNET_BRANCH=${{ matrix.mainnet_branch }}93            POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }}94            WASM_NAME=${{ matrix.wasm_name }}95            POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}9697      - name: Show build configuration98        run: cat .docker/Dockerfile-parachain-node-only.${{ matrix.network }}.yml99100      - name: Generate launch-config.json101        uses: cuchi/jinja2-action@v1.2.0102        with:103          template: .docker/forkless-config/launch-config-forkless-nodata.j2104          output_file: .docker/forkless-config/launch-config-node-only.json105          variables: |106            WASM_NAME=${{ matrix.wasm_name }}107            RELAY_CHAIN_TYPE=${{ env.RELAY_CHAIN_TYPE }}108109      - name: Show launch-config-forkless configuration110        run: cat .docker/forkless-config/launch-config-node-only.json111112      - name: Run find-and-replace to remove slashes from branch name113        uses: mad9000/actions-find-and-replace-string@4114        id: branchname115        with:116          source: ${{ github.head_ref }}117          find: '/'118          replace: '-'119120      - name: Set build SHA121        shell: bash122        run: |123          echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV124125      - name: Build the stack126        run: cd .docker/ && docker build --no-cache --file ./Dockerfile-parachain-node-only.${{ matrix.network }}.yml --tag uniquenetwork/ci-node-only-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA ../127128      - name: Log in to Docker Hub129        uses: docker/login-action@v2.1.0130        with:131          username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}132          password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}133134      - name: Push docker image version135        run: docker push uniquenetwork/ci-node-only-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA136137      - name: Remove builder cache138        if: always()                   # run this step always139        run: |140          docker builder prune -f141          docker system prune -f          142143144  node-only-update-tests:145    needs: [prepare-execution-matrix, node-only-update-build]146    # The type of runner that the job will run on147    runs-on: [self-hosted-ci, large]148149    timeout-minutes: 2880150151    name: ${{ matrix.network }}-tests152153    continue-on-error: true         #Do not stop testing of matrix runs failed.  As it decided during PR review - it required 50/50& Let's check it with false.154155    strategy:156      matrix:157        include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}}158159    steps:160      - name: Skip if pull request is in Draft161        if: github.event.pull_request.draft == true162        run: exit 1163164      - name: Clean Workspace165        uses: AutoModality/action-clean@v1.1.0166167      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it168      - uses: actions/checkout@v3.1.0169        with:170          ref: ${{ github.head_ref }}  #Checking out head commit171172      - name: Prepare173        uses: ./.github/actions/prepare174175      - name: Set build SHA176        shell: bash177        run: |178          echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV179180      - name: Run find-and-replace to remove slashes from branch name181        uses: mad9000/actions-find-and-replace-string@4182        id: branchname183        with:184          source: ${{ github.head_ref }}185          find: '/'186          replace: '-'187188      - name: Read .env file189        uses: xom9ikk/dotenv@v2190191      - name: Generate ENV related extend file for docker-compose192        uses: cuchi/jinja2-action@v1.2.0193        with:194          template: .docker/docker-compose.node-only.j2195          output_file: .docker/docker-compose.node-only.${{ matrix.network }}.yml196          variables: |197            NETWORK=${{ matrix.network }}198            BUILD_TAG=${{ steps.branchname.outputs.value }}-$BUILD_SHA199200      - name: Show build configuration201        run: cat .docker/docker-compose.node-only.${{ matrix.network }}.yml202203      - name: Log in to Docker Hub204        uses: docker/login-action@v2.1.0205        with:206          username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}207          password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}208209      - name: Build the stack210        run: docker-compose -f ".docker/docker-compose.node-only.${{ matrix.network }}.yml" up -d --remove-orphans --force-recreate --timeout 300211212      #  🚀 POLKADOT LAUNCH COMPLETE 🚀213      - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade.214        if: success()215        run: |216          counter=160217          function check_container_status {218                docker inspect -f {{.State.Running}} node-only219          }220          function do_docker_logs {221                docker logs --details node-only  2>&1222          }223          function is_started {224                if [ "$(check_container_status)" == "true" ]; then225                        echo "Container: node-only RUNNING";226                        echo "Check Docker logs"227                        DOCKER_LOGS=$(do_docker_logs)228                        if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then229                                echo "🚀 POLKADOT LAUNCH COMPLETE 🚀"230                                return 0231                        else232                                echo "Message not found in logs output, repeating..."233                                return 1234                        fi235                else236                        echo "Container node-only NOT RUNNING"237                        echo "Halting all future checks"238                        exit 1239                fi240          echo "something goes wrong"241          exit 1242          }243          while ! is_started; do244                echo "Waiting for special message in log files "245                sleep 30s246                counter=$(( $counter - 1 ))247                echo "Counter: $counter"248                if [ "$counter" -gt "0" ]; then249                         continue250                else251                         break252                fi253          done254          echo "Halting script"255          exit 0256        shell: bash257258      - name: Checkout at '${{ matrix.mainnet_branch }}' branch259        uses: actions/checkout@master260        with:261          ref: ${{ matrix.mainnet_branch }}  #Checking out head commit262          path: ${{ matrix.mainnet_branch }}263264      - uses: actions/setup-node@v3.5.1265        with:266          node-version: 16267268      - name: Run Parallel tests before Node Parachain upgrade269        working-directory: ${{ matrix.mainnet_branch }}/tests270        if: success() || failure()271        run: |272          yarn install273          yarn add mochawesome274          ./scripts/wait_for_first_block.sh275          echo "Ready to start tests"276          yarn polkadot-types277          NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW}278        env:279          RPC_URL: http://127.0.0.1:9944/280281      - name: Run Sequential tests before Node Parachain upgrade282        if: success() || failure()283        working-directory: ${{ matrix.mainnet_branch }}/tests284        run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW}285        env:286          RPC_URL: http://127.0.0.1:9944/287288      - name: Send SIGUSR1 to polkadot-launch process289        if: success() || failure()290        run: |291          #Get PID of polkadot-launch292          ContainerID=$(docker ps -aqf "name=node-only")293          PID=$(docker exec node-only pidof 'polkadot-launch')294          sleep 30s295          echo -e "\n"296          echo -e "Restart polkadot-launch process: $PID\n"297          docker exec node-only kill -SIGUSR1 ${PID}298          echo "SIGUSR1 sent to Polkadot-launch PID: $PID"299          sleep 60s300          echo -e "Show logs of node-only container.\n"301          docker logs ${ContainerID}302303      - name: Get chain logs in case of docker image crashed after Polkadot Launch restart304        if: failure()                   # run this step only at failure305        run: |306          docker exec node-only tail -n 1000 /polkadot-launch/9944.log307          docker exec node-only tail -n 1000 /polkadot-launch/9945.log308          docker exec node-only tail -n 1000 /polkadot-launch/alice.log309310      - name: copy chain log files from container to the host311        if: success() || failure()    # run this step even if previous step failed312        run: |313          mkdir -p /tmp/node-only-update314          docker cp node-only:/polkadot-launch/9944.log /tmp/node-only-update/315          docker cp node-only:/polkadot-launch/9945.log /tmp/node-only-update/316          docker cp node-only:/polkadot-launch/alice.log /tmp/node-only-update/317318      - name: Upload chain log files319        if: success() || failure()320        uses: actions/upload-artifact@v3321        with:322          name: node-only-update-chain-logs323          path: /tmp/node-only-update/324          if-no-files-found: warn325326      - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade.327        if: success()328        run: |329          counter=160330          function check_container_status {331                docker inspect -f {{.State.Running}} node-only332          }333          function do_docker_logs {334                docker logs --details node-only  2>&1335          }336          function is_started {337                if [ "$(check_container_status)" == "true" ]; then338                        echo "Container: node-only RUNNING";339                        echo "Check Docker logs"340                        DOCKER_LOGS=$(do_docker_logs)341                        if [[ ${DOCKER_LOGS} = *"All parachain collators restarted with the new binaries."* ]];then342                                echo "🌗 All parachain collators restarted with the new binaries."343                                return 0344                        else345                                echo "Message not found in logs output, repeating..."346                                return 1347                        fi348                else349                        echo "Container node-only NOT RUNNING"350                        echo "Halting all future checks"351                        exit 1352                fi353          echo "something goes wrong"354          exit 1355          }356          while ! is_started; do357                echo "Waiting for special message in log files "358                sleep 30s359                counter=$(( $counter - 1 ))360                echo "Counter: $counter"361                if [ "$counter" -gt "0" ]; then362                         continue363                else364                         break365                fi366          done367          echo "Halting script"368          exit 0369        shell: bash370371      - name: Run Parallel tests after Node Parachain upgrade372        working-directory: ${{ matrix.mainnet_branch }}/tests373        if: success() || failure()    # run this step even if previous step failed374        run: |375          yarn install376          yarn add mochawesome377          ./scripts/wait_for_first_block.sh378          echo "Ready to start tests"379          yarn polkadot-types380          NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW}381        env:382          RPC_URL: http://127.0.0.1:9944/383384      - name: Run Sequential tests after Node Parachain upgrade385        if: success() || failure()386        working-directory: ${{ matrix.mainnet_branch }}/tests387        run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW}388        env:389          RPC_URL: http://127.0.0.1:9944/390391      - name: Stop running containers392        if: always()                   # run this step always393        run: docker-compose -f ".docker/docker-compose.node-only.${{ matrix.network }}.yml" down394395      - name: Remove builder cache396        if: always()                   # run this step always397        run: |398          docker builder prune -f -a399          docker system prune -f400          docker image prune -f -a401402      - name: Remove repo at the end403        if: always()                   # run this step always404        run: |405          ls -ls ./