From e7df43a9b6d1d4f387c331a32ad2aed86836b46d Mon Sep 17 00:00:00 2001 From: BuddyGlas <104383456+BuddyGlas@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:51:04 +0000 Subject: [PATCH] ci: node only update changes (#956) --- --- a/.docker/Dockerfile-parachain-node-only +++ /dev/null @@ -1,83 +0,0 @@ -ARG POLKADOT_BUILD_BRANCH -FROM uniquenetwork/builder-polkadot:${POLKADOT_BUILD_BRANCH} as polkadot - -# ===== Rust builder ===== -FROM uniquenetwork/services:latest as rust-builder - -ARG RUST_TOOLCHAIN= -ENV RUST_TOOLCHAIN $RUST_TOOLCHAIN -ENV CARGO_HOME="/cargo-home" -ENV PATH="/cargo-home/bin:$PATH" -ENV TZ=UTC -RUN rustup toolchain uninstall $(rustup toolchain list) && \ - rustup toolchain install $RUST_TOOLCHAIN && \ - rustup default $RUST_TOOLCHAIN && \ - rustup target list --installed && \ - rustup show -RUN rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN - -RUN mkdir /unique_parachain -WORKDIR /unique_parachain - - -# ===== BUILD current version ====== -FROM rust-builder as builder-unique-current - -ARG PROFILE=release -ARG NETWORK -ARG MAINNET_BRANCH -ARG REPO_URL - -WORKDIR /unique_parachain - -RUN git clone $REPO_URL -b $MAINNET_BRANCH . && \ - cargo build --features=${NETWORK}-runtime --$PROFILE - -# ===== BUILD target version ====== -FROM rust-builder as builder-unique-target - -ARG PROFILE=release -ARG NETWORK - -COPY . /unique_parachain -WORKDIR /unique_parachain - -RUN cargo build --features=${NETWORK}-runtime --$PROFILE - -# ===== RUN ====== - -FROM ubuntu:22.04 - -ARG NETWORK -ARG POLKADOT_LAUNCH_BRANCH -ARG WASM_NAME - -RUN apt-get -y update && \ - apt-get -y install curl git && \ - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \ - export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - nvm install v16.16.0 && \ - nvm use v16.16.0 - -RUN git clone https://github.com/uniquenetwork/polkadot-launch -b ${POLKADOT_LAUNCH_BRANCH} - -RUN export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - cd /polkadot-launch && \ - npm install --global yarn && \ - yarn install - -RUN echo "$NETWORK" - -COPY --from=builder-unique-current /unique_parachain/target/release/unique-collator /unique-chain/current/release/ -COPY --from=builder-unique-target /unique_parachain/target/release/unique-collator /unique-chain/target/release/ -COPY --from=builder-unique-target /unique_parachain/target/release/wbuild/"$WASM_NAME"-runtime/"$WASM_NAME"_runtime.compact.compressed.wasm /unique-chain/target/release/wbuild/"$WASM_NAME"-runtime/"$WASM_NAME"_runtime.compact.compressed.wasm - -COPY --from=polkadot /unique_parachain/polkadot/target/release/polkadot /polkadot/target/release/ -COPY --from=polkadot /unique_parachain/polkadot/target/release/wbuild/westend-runtime/westend_runtime.compact.compressed.wasm /polkadot/target/release/wbuild/westend-runtime/westend_runtime.compact.compressed.wasm - -CMD export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - cd /polkadot-launch && \ - yarn start launch-config.json --test-upgrade-parachains -w -n --- /dev/null +++ b/.docker/Dockerfile-parachain-node-only.j2 @@ -0,0 +1,67 @@ +# ===== Rust builder ===== +FROM uniquenetwork/services:latest as rust-builder +ENV CARGO_HOME="/cargo-home" +ENV PATH="/cargo-home/bin:$PATH" +ENV TZ=UTC + +RUN rustup toolchain uninstall $(rustup toolchain list) && \ + rustup toolchain install {{ RUST_TOOLCHAIN }} && \ + rustup default {{ RUST_TOOLCHAIN }} && \ + rustup target list --installed && \ + rustup show +RUN rustup target add wasm32-unknown-unknown --toolchain {{ RUST_TOOLCHAIN }} + +RUN mkdir /unique_parachain +WORKDIR /unique_parachain + + +# ===== BUILD current version ====== +FROM rust-builder as builder-unique-current + +ARG PROFILE=release + +WORKDIR /unique_parachain + +RUN git clone -b {{ MAINNET_BRANCH }} https://github.com/UniqueNetwork/unique-chain.git . && \ + cargo build --features={{ NETWORK }}-runtime --$PROFILE + +# ===== BUILD target version ====== +FROM rust-builder as builder-unique-target + +ARG PROFILE=release + +COPY . /unique_parachain +WORKDIR /unique_parachain + +RUN cargo build --features={{ NETWORK }}-runtime --$PROFILE + +# ===== RUN ====== + +FROM ubuntu:22.04 + +RUN apt-get -y update && \ + apt-get -y install curl git && \ + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \ + export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + nvm install v16.16.0 && \ + nvm use v16.16.0 + +RUN git clone https://github.com/uniquenetwork/polkadot-launch -b {{ POLKADOT_LAUNCH_BRANCH }} + +COPY --from=builder-unique-current /unique_parachain/target/release/unique-collator /unique-chain/current/release/ + +COPY --from=builder-unique-target /unique_parachain/target/release/unique-collator /unique-chain/target/release/ +COPY --from=builder-unique-target /unique_parachain/target/release/wbuild/{{ WASM_NAME }}-runtime/{{ WASM_NAME }}_runtime.compact.compressed.wasm /unique-chain/target/release/wbuild/{{ WASM_NAME }}-runtime/{{ WASM_NAME }}_runtime.compact.compressed.wasm +COPY --from=builder-unique-target /unique_parachain/.docker/forkless-config/launch-config-node-only.json /polkadot-launch/launch-config.json + +COPY --from=uniquenetwork/builder-polkadot:{{ POLKADOT_BUILD_BRANCH }} /unique_parachain/polkadot/target/release/polkadot /polkadot/target/release/ +COPY --from=uniquenetwork/builder-polkadot:{{ POLKADOT_BUILD_BRANCH }} /unique_parachain/polkadot/target/release/wbuild/westend-runtime/westend_runtime.compact.compressed.wasm /polkadot/target/release/wbuild/westend-runtime/westend_runtime.compact.compressed.wasm +COPY --from=uniquenetwork/builder-polkadot:{{ POLKADOT_BUILD_BRANCH }} /unique_parachain/polkadot/target/release/wbuild/rococo-runtime/rococo_runtime.compact.compressed.wasm /polkadot/target/release/wbuild/rococo-runtime/rococo_runtime.compact.compressed.wasm + +CMD export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + cd /polkadot-launch && \ + npm install --global yarn && \ + yarn install && \ + yarn start launch-config.json --test-upgrade-parachains -w -n --- /dev/null +++ b/.docker/docker-compose.node-only.j2 @@ -0,0 +1,16 @@ +version: "3.5" + +services: + node-only: + image: uniquenetwork/ci-node-only-local:{{ NETWORK }}-{{ BUILD_TAG }} + container_name: node-only + expose: + - 9944 + - 9945 + - 9933 + - 9844 + ports: + - 127.0.0.1:9944:9944 + - 127.0.0.1:9945:9945 + - 127.0.0.1:9933:9933 + - 127.0.0.1:9844:9844 --- a/.github/workflows/node-only-update.yml +++ b/.github/workflows/node-only-update.yml @@ -11,7 +11,7 @@ # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - nodes-execution-matrix: + prepare-execution-matrix: name: execution matrix @@ -43,23 +43,28 @@ network {unique}, wasm_name {unique}, mainnet_branch {${{ env.UNIQUE_MAINNET_BRANCH }}}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}} - nodes-only-update: - needs: nodes-execution-matrix + node-only-update-build: + + needs: prepare-execution-matrix # The type of runner that the job will run on runs-on: [self-hosted-ci] timeout-minutes: 2880 # 48 hours for execution jobs. - name: ${{ matrix.network }} + name: ${{ matrix.network }}-build continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. strategy: matrix: - include: ${{fromJson(needs.nodes-execution-matrix.outputs.matrix)}} + include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}} steps: + - name: Skip if pull request is in Draft + if: github.event.pull_request.draft == true + run: exit 1 + - name: Clean Workspace uses: AutoModality/action-clean@v1.1.0 @@ -68,48 +73,141 @@ with: ref: ${{ github.head_ref }} #Checking out head commit + # Prepare SHA + - name: Prepare SHA + uses: ./.github/actions/prepare + - name: Read .env file uses: xom9ikk/dotenv@v2 - - name: Generate ENV related extend file for docker-compose + # Build main image for NODE-ONLY-UPDATE + - name: Generate ENV related extend Dockerfile file uses: cuchi/jinja2-action@v1.2.0 with: - template: .docker/docker-compose.tmp-node.j2 - output_file: .docker/docker-compose.node.${{ matrix.network }}.yml + template: .docker/Dockerfile-parachain-node-only.j2 + output_file: .docker/Dockerfile-parachain-node-only.${{ matrix.network }}.yml variables: | - REPO_URL=${{ github.server_url }}/${{ github.repository }}.git RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} - POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }} - POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }} - # POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }} - # MAINNET_TAG=${{ matrix.mainnet_tag }} + NETWORK=${{ matrix.network }} MAINNET_BRANCH=${{ matrix.mainnet_branch }} - NETWORK=${{ matrix.network }} + POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }} WASM_NAME=${{ matrix.wasm_name }} - # BRANCH=${{ github.head_ref }} + POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }} - name: Show build configuration - run: cat .docker/docker-compose.node.${{ matrix.network }}.yml + run: cat .docker/Dockerfile-parachain-node-only.${{ matrix.network }}.yml - - name: Generate launch-config-forkless-nodata.json + - name: Generate launch-config.json uses: cuchi/jinja2-action@v1.2.0 with: template: .docker/forkless-config/launch-config-forkless-nodata.j2 - output_file: .docker/launch-config-forkless-nodata.json + output_file: .docker/forkless-config/launch-config-node-only.json variables: | - NETWORK=${{ matrix.network }} WASM_NAME=${{ matrix.wasm_name }} RELAY_CHAIN_TYPE=${{ env.RELAY_CHAIN_TYPE }} - name: Show launch-config-forkless configuration - run: cat .docker/launch-config-forkless-nodata.json + run: cat .docker/forkless-config/launch-config-node-only.json + + - name: Run find-and-replace to remove slashes from branch name + uses: mad9000/actions-find-and-replace-string@4 + id: branchname + with: + source: ${{ github.head_ref }} + find: '/' + replace: '-' + + - name: Set build SHA + shell: bash + run: | + echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV + + - name: Build the stack + run: cd .docker/ && docker build --no-cache --file ./Dockerfile-parachain-node-only.${{ matrix.network }}.yml --tag uniquenetwork/ci-node-only-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA ../ + + - name: Log in to Docker Hub + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} + password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} + + - name: Push docker image version + run: docker push uniquenetwork/ci-node-only-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA + + - name: Remove builder cache + if: always() # run this step always + run: | + docker builder prune -f + docker system prune -f + + + node-only-update-tests: + needs: [prepare-execution-matrix, node-only-update-build] + # The type of runner that the job will run on + runs-on: [self-hosted-ci, large] + + timeout-minutes: 2880 + + name: ${{ matrix.network }}-tests + + continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. + + strategy: + matrix: + include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}} + + steps: + - name: Skip if pull request is in Draft + if: github.event.pull_request.draft == true + run: exit 1 + + - name: Clean Workspace + uses: AutoModality/action-clean@v1.1.0 + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3.1.0 + with: + ref: ${{ github.head_ref }} #Checking out head commit + + - name: Prepare + uses: ./.github/actions/prepare + + - name: Set build SHA + shell: bash + run: | + echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV + + - name: Run find-and-replace to remove slashes from branch name + uses: mad9000/actions-find-and-replace-string@4 + id: branchname + with: + source: ${{ github.head_ref }} + find: '/' + replace: '-' - - uses: actions/setup-node@v3.5.1 + - name: Read .env file + uses: xom9ikk/dotenv@v2 + + - name: Generate ENV related extend file for docker-compose + uses: cuchi/jinja2-action@v1.2.0 with: - node-version: 16 + template: .docker/docker-compose.node-only.j2 + output_file: .docker/docker-compose.node-only.${{ matrix.network }}.yml + variables: | + NETWORK=${{ matrix.network }} + BUILD_TAG=${{ steps.branchname.outputs.value }}-$BUILD_SHA + + - name: Show build configuration + run: cat .docker/docker-compose.node-only.${{ matrix.network }}.yml + + - name: Log in to Docker Hub + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} + password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - name: Build the stack - run: docker-compose -f ".docker/docker-compose.node.${{ matrix.network }}.yml" up -d --build --remove-orphans --force-recreate --timeout 300 + run: docker-compose -f ".docker/docker-compose.node-only.${{ matrix.network }}.yml" up -d --remove-orphans --force-recreate --timeout 300 # 🚀 POLKADOT LAUNCH COMPLETE 🚀 - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade. @@ -117,14 +215,14 @@ run: | counter=160 function check_container_status { - docker inspect -f {{.State.Running}} node-parachain + docker inspect -f {{.State.Running}} node-only } function do_docker_logs { - docker logs --details node-parachain 2>&1 + docker logs --details node-only 2>&1 } function is_started { if [ "$(check_container_status)" == "true" ]; then - echo "Container: node-parachain RUNNING"; + echo "Container: node-only RUNNING"; echo "Check Docker logs" DOCKER_LOGS=$(do_docker_logs) if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then @@ -135,7 +233,7 @@ return 1 fi else - echo "Container node-parachain NOT RUNNING" + echo "Container node-only NOT RUNNING" echo "Halting all future checks" exit 1 fi @@ -163,6 +261,10 @@ ref: ${{ matrix.mainnet_branch }} #Checking out head commit path: ${{ matrix.mainnet_branch }} + - uses: actions/setup-node@v3.5.1 + with: + node-version: 16 + - name: Run Parallel tests before Node Parachain upgrade working-directory: ${{ matrix.mainnet_branch }}/tests if: success() || failure() @@ -176,16 +278,6 @@ env: RPC_URL: http://127.0.0.1:9944/ - # - name: Upload Parallel Test Report Before Node upgrade - # uses: phoenix-actions/test-reporting@v8 - # id: test-parallel-report-before - # if: success() || failure() # run this step even if previous step failed - # with: - # name: Tests before node upgrade ${{ matrix.network }} # Name of the check run which will be created - # path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-parallel-*.json # Path to test results - # reporter: mochawesome-json - # fail-on-error: 'false' - - name: Run Sequential tests before Node Parachain upgrade if: success() || failure() working-directory: ${{ matrix.mainnet_branch }}/tests @@ -193,45 +285,35 @@ env: RPC_URL: http://127.0.0.1:9944/ - # - name: Upload Sequential Test Report Before Node upgrade - # uses: phoenix-actions/test-reporting@v10 - # id: test-sequential-report-before - # if: success() || failure() # run this step even if previous step failed - # with: - # name: Tests before node upgrade ${{ matrix.network }} # Name of the check run which will be created - # path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-sequential-*.json # Path to test results - # reporter: mochawesome-json - # fail-on-error: 'false' - - name: Send SIGUSR1 to polkadot-launch process if: success() || failure() run: | #Get PID of polkadot-launch - ContainerID=$(docker ps -aqf "name=node-parachain") - PID=$(docker exec node-parachain pidof 'polkadot-launch') + ContainerID=$(docker ps -aqf "name=node-only") + PID=$(docker exec node-only pidof 'polkadot-launch') sleep 30s echo -e "\n" echo -e "Restart polkadot-launch process: $PID\n" - docker exec node-parachain kill -SIGUSR1 ${PID} + docker exec node-only kill -SIGUSR1 ${PID} echo "SIGUSR1 sent to Polkadot-launch PID: $PID" sleep 60s - echo -e "Show logs of node-parachain container.\n" + echo -e "Show logs of node-only container.\n" docker logs ${ContainerID} - name: Get chain logs in case of docker image crashed after Polkadot Launch restart if: failure() # run this step only at failure run: | - docker exec node-parachain tail -n 1000 /polkadot-launch/9944.log - docker exec node-parachain tail -n 1000 /polkadot-launch/9945.log - docker exec node-parachain tail -n 1000 /polkadot-launch/alice.log + docker exec node-only tail -n 1000 /polkadot-launch/9944.log + docker exec node-only tail -n 1000 /polkadot-launch/9945.log + docker exec node-only tail -n 1000 /polkadot-launch/alice.log - name: copy chain log files from container to the host if: success() || failure() # run this step even if previous step failed run: | mkdir -p /tmp/node-only-update - docker cp node-parachain:/polkadot-launch/9944.log /tmp/node-only-update/ - docker cp node-parachain:/polkadot-launch/9945.log /tmp/node-only-update/ - docker cp node-parachain:/polkadot-launch/alice.log /tmp/node-only-update/ + docker cp node-only:/polkadot-launch/9944.log /tmp/node-only-update/ + docker cp node-only:/polkadot-launch/9945.log /tmp/node-only-update/ + docker cp node-only:/polkadot-launch/alice.log /tmp/node-only-update/ - name: Upload chain log files if: success() || failure() @@ -246,14 +328,14 @@ run: | counter=160 function check_container_status { - docker inspect -f {{.State.Running}} node-parachain + docker inspect -f {{.State.Running}} node-only } function do_docker_logs { - docker logs --details node-parachain 2>&1 + docker logs --details node-only 2>&1 } function is_started { if [ "$(check_container_status)" == "true" ]; then - echo "Container: node-parachain RUNNING"; + echo "Container: node-only RUNNING"; echo "Check Docker logs" DOCKER_LOGS=$(do_docker_logs) if [[ ${DOCKER_LOGS} = *"All parachain collators restarted with the new binaries."* ]];then @@ -264,7 +346,7 @@ return 1 fi else - echo "Container node-parachain NOT RUNNING" + echo "Container node-only NOT RUNNING" echo "Halting all future checks" exit 1 fi @@ -298,16 +380,6 @@ NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW} env: RPC_URL: http://127.0.0.1:9944/ - - #- name: Test Report Parallel After Node upgrade - # uses: phoenix-actions/test-reporting@v10 - # id: test-report-parallel-after - # if: success() || failure() # run this step even if previous step failed - # with: - # name: Tests after node upgrade ${{ matrix.network }} # Name of the check run which will be created - # path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-parallel-*.json # Path to test results - # reporter: mochawesome-json - # fail-on-error: 'false' - name: Run Sequential tests after Node Parachain upgrade if: success() || failure() @@ -315,20 +387,10 @@ run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW} env: RPC_URL: http://127.0.0.1:9944/ - - #- name: Upload Sequential Test Report After Node upgrade - # uses: phoenix-actions/test-reporting@v10 - # id: test-sequential-report-after - # if: success() || failure() # run this step even if previous step failed - # with: - # name: Tests before node upgrade ${{ matrix.network }} # Name of the check run which will be created - # path: ${{ matrix.mainnet_branch }}/tests/mochawesome-report/test-sequential-*.json # Path to test results - # reporter: mochawesome-json - # fail-on-error: 'false' - name: Stop running containers if: always() # run this step always - run: docker-compose -f ".docker/docker-compose.node.${{ matrix.network }}.yml" down --volumes + run: docker-compose -f ".docker/docker-compose.node-only.${{ matrix.network }}.yml" down - name: Remove builder cache if: always() # run this step always -- gitstuff