From 6f79f09ea99780523ec9370ab236bdd4414233fd Mon Sep 17 00:00:00 2001 From: Konstantin Astakhov Date: Fri, 23 Jun 2023 06:12:29 +0000 Subject: [PATCH] ci collator-selection for dockerhub --- --- a/.docker/Dockerfile-collators +++ /dev/null @@ -1,74 +0,0 @@ -ARG POLKADOT_BUILD_BRANCH -FROM uniquenetwork/builder-polkadot:${POLKADOT_BUILD_BRANCH} as polkadot - -# ===== Rust builder ===== -FROM uniquenetwork/services:latest as rust-builder -ARG RUST_TOOLCHAIN= - -ENV RUST_TOOLCHAIN $RUST_TOOLCHAIN -ENV CARGO_HOME="/cargo-home" -ENV PATH="/cargo-home/bin:$PATH" -ENV TZ=UTC - -RUN rustup toolchain uninstall $(rustup toolchain list) && \ - rustup toolchain install $RUST_TOOLCHAIN && \ - rustup default $RUST_TOOLCHAIN && \ - rustup target list --installed && \ - rustup show -RUN rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN - -RUN mkdir /unique_parachain -WORKDIR /unique_parachain - -# ===== BUILD ====== -FROM rust-builder as builder-unique - -ARG PROFILE=release -ARG NETWORK -ARG POLKADOT_LAUNCH_BRANCH -ARG BRANCH - -ENV POLKADOT_LAUNCH_BRANCH $POLKADOT_LAUNCH_BRANCH - -WORKDIR /unique_parachain - -RUN git clone -b $BRANCH https://github.com/UniqueNetwork/unique-chain.git && \ - cd unique-chain && \ - cargo build --features=$NETWORK-runtime --$PROFILE - -# ===== RUN ====== - -FROM ubuntu:22.04 - -RUN apt-get -y update && \ - apt-get -y install curl git && \ - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \ - export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - nvm install v16.16.0 && \ - nvm use v16.16.0 - -RUN git clone https://github.com/uniquenetwork/polkadot-launch -b ${POLKADOT_LAUNCH_BRANCH} - -RUN export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - cd /polkadot-launch && \ - npm install --global yarn && \ - yarn install - -COPY --from=builder-unique /unique_parachain/unique-chain/.docker/testnet-config/launch-config.json /polkadot-launch/launch-config.json -COPY --from=builder-unique /unique_parachain/unique-chain/target/release/unique-collator /unique-chain/target/release/ - -COPY --from=polkadot /unique_parachain/polkadot/target/release/polkadot /polkadot/target/release/ - -EXPOSE 9844 -EXPOSE 9944 -EXPOSE 9933 -EXPOSE 9833 -EXPOSE 40333 -EXPOSE 30333 - -CMD export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - cd /polkadot-launch && \ - yarn start launch-config.json --- /dev/null +++ b/.docker/Dockerfile-collators.j2 @@ -0,0 +1,64 @@ +FROM uniquenetwork/builder-polkadot:{{ POLKADOT_BUILD_BRANCH }} as polkadot + +# ===== Rust builder ===== +FROM uniquenetwork/services:latest as rust-builder + +ENV CARGO_HOME="/cargo-home" +ENV PATH="/cargo-home/bin:$PATH" +ENV TZ=UTC +RUN rustup toolchain uninstall $(rustup toolchain list) && \ + rustup toolchain install {{ RUST_TOOLCHAIN }} && \ + rustup default {{ RUST_TOOLCHAIN }} && \ + rustup target list --installed && \ + rustup show +RUN rustup target add wasm32-unknown-unknown --toolchain {{ RUST_TOOLCHAIN }} + +RUN mkdir /unique_parachain +WORKDIR /unique_parachain + +# ===== BUILD ====== +FROM rust-builder as builder-unique + +WORKDIR /unique_parachain + +ARG PROFILE=release + +RUN git clone https://github.com/UniqueNetwork/unique-chain.git -b {{ BRANCH }} . && \ + cargo build --features={{ NETWORK }}-runtime --$PROFILE + +# ===== RUN ====== + +FROM ubuntu:22.04 + +RUN apt-get -y update && \ + apt-get -y install curl git && \ + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \ + export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + nvm install v16.16.0 && \ + nvm use v16.16.0 + +RUN git clone https://github.com/uniquenetwork/polkadot-launch -b {{ POLKADOT_LAUNCH_BRANCH }} + +RUN export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + cd /polkadot-launch && \ + npm install --global yarn && \ + yarn install + +COPY --from=builder-unique /unique_parachain/target/release/unique-collator /unique-chain/target/release/ +COPY --from=builder-unique /unique_parachain/launch-config.json /polkadot-launch/launch-config.json + +COPY --from=polkadot /unique_parachain/polkadot/target/release/polkadot /polkadot/target/release/ + +EXPOSE 9844 +EXPOSE 9944 +EXPOSE 9933 +EXPOSE 9833 +EXPOSE 40333 +EXPOSE 30333 + +CMD export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + cd /polkadot-launch && \ + yarn start launch-config.json \ No newline at end of file --- /dev/null +++ b/.docker/docker-compose.collators.j2 @@ -0,0 +1,12 @@ +version: "3.5" + +services: + forkless-data: + image: uniquenetwork/ci-collator-selection-local:{{ NETWORK }}-{{ BUILD_TAG }} + container_name: collator-selection-{{ NETWORK }} + expose: + - 9944 + - 9933 + ports: + - 127.0.0.1:9944:9944 + - 127.0.0.1:9933:9933 --- a/.docker/docker-compose.tmp-collators.j2 +++ /dev/null @@ -1,25 +0,0 @@ -version: "3.5" - -services: - parachain-collators: - build: - args: - - "RUST_TOOLCHAIN={{ RUST_TOOLCHAIN }}" - - "POLKADOT_BUILD_BRANCH={{ POLKADOT_BUILD_BRANCH }}" - - "POLKADOT_LAUNCH_BRANCH={{ POLKADOT_LAUNCH_BRANCH }}" - - "NETWORK={{ NETWORK }}" - - "BRANCH={{ BRANCH }}" - context: ../ - dockerfile: .docker/Dockerfile-parachain - image: parachain-collators - container_name: parachain-collators - expose: - - 9944 - - 9933 - ports: - - 127.0.0.1:9944:9944 - - 127.0.0.1:9933:9933 - logging: - options: - max-size: "1m" - max-file: "3" --- a/.github/workflows/collator-selection.yml +++ b/.github/workflows/collator-selection.yml @@ -6,7 +6,7 @@ # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - execution-marix: + prepare-execution-marix: name: Prepare execution matrix runs-on: self-hosted-ci @@ -35,16 +35,16 @@ network {quartz}, wasm_name {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, replica_from_address {${{ env.QUARTZ_REPLICA_FROM }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}} network {sapphire}, wasm_name {quartz}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, replica_from_address {${{ env.SAPPHIRE_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}} - collator-selection: - needs: execution-marix + collator-selection-build: + needs: prepare-execution-marix # The type of runner that the job will run on runs-on: [self-hosted-ci,large] timeout-minutes: 1380 - name: ${{ matrix.network }} + name: ${{ matrix.network }}-build strategy: matrix: - include: ${{fromJson(needs.execution-marix.outputs.matrix)}} + include: ${{fromJson(needs.prepare-execution-marix.outputs.matrix)}} continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. @@ -60,23 +60,121 @@ - name: Read .env file uses: xom9ikk/dotenv@v2 - - name: Generate ENV related extend file for docker-compose + # Prepare SHA + - name: Prepare SHA + uses: ./.github/actions/prepare + + # Build main image for FORKLESS-UPDATE-NODATA + - name: Generate ENV related extend Dockerfile file uses: cuchi/jinja2-action@v1.2.0 with: - template: .docker/docker-compose.tmp-collators.j2 - output_file: .docker/docker-compose.collators.${{ matrix.network }}.yml + template: .docker/Dockerfile-collators.j2 + output_file: .docker/Dockerfile-collators.${{ matrix.network }}.yml variables: | RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} + NETWORK=${{ matrix.network }} + POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }} POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }} - POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }} - NETWORK=${{ matrix.network }} BRANCH=${{ github.head_ref }} - name: Show build configuration + run: cat .docker/Dockerfile-collators.${{ matrix.network }}.yml + + - name: Run find-and-replace to remove slashes from branch name + uses: mad9000/actions-find-and-replace-string@4 + id: branchname + with: + source: ${{ github.head_ref }} + find: '/' + replace: '-' + + - name: Set build SHA + shell: bash + run: | + echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV + + - name: Build the stack + run: cd .docker/ && docker build --no-cache --file ./Dockerfile-collators.${{ matrix.network }}.yml --tag uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA ../ + + - name: Log in to Docker Hub + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} + password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} + + - name: Push docker image version + run: docker push uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA + + - name: Remove builder cache + if: always() # run this step always + run: | + docker builder prune -f + docker system prune -f + + collator-selection-tests: + needs: [prepare-execution-marix, collator-selection-build] + # The type of runner that the job will run on + runs-on: [self-hosted-ci, large] + + timeout-minutes: 600 + + name: ${{ matrix.network }}-tests + + continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. + + strategy: + matrix: + include: ${{fromJson(needs.prepare-execution-marix.outputs.matrix)}} + + steps: + - name: Skip if pull request is in Draft + if: github.event.pull_request.draft == true + run: exit 1 + + - name: Clean Workspace + uses: AutoModality/action-clean@v1.1.0 + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3.1.0 + with: + ref: ${{ github.head_ref }} #Checking out head commit + + - name: Prepare + uses: ./.github/actions/prepare + + - name: Set build SHA + shell: bash + run: | + echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV + + - name: Run find-and-replace to remove slashes from branch name + uses: mad9000/actions-find-and-replace-string@4 + id: branchname + with: + source: ${{ github.head_ref }} + find: '/' + replace: '-' + + - name: Read .env file + uses: xom9ikk/dotenv@v2 + + - name: Generate ENV related extend file for docker-compose + uses: cuchi/jinja2-action@v1.2.0 + with: + template: .docker/docker-compose.collators.j2 + output_file: .docker/docker-compose.collators.${{ matrix.network }}.yml + variables: | + NETWORK=${{ matrix.network }} + BUILD_TAG=${{ steps.branchname.outputs.value }}-$BUILD_SHA + + - name: Show build configuration run: cat .docker/docker-compose.collators.${{ matrix.network }}.yml - - name: Show launch-config configuration - run: cat launch-config.json + - name: Log in to Docker Hub + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} + password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - name: Build the stack run: docker-compose -f ".docker/docker-compose.collators.${{ matrix.network }}.yml" up -d --remove-orphans --force-recreate --timeout 300 @@ -87,14 +185,14 @@ run: | counter=160 function check_container_status { - docker inspect -f {{.State.Running}} parachain-collators + docker inspect -f {{.State.Running}} collator-selection-${{ matrix.network }} } function do_docker_logs { - docker logs --details parachain-collators 2>&1 + docker logs --details collator-selection-${{ matrix.network }} 2>&1 } function is_started { if [ "$(check_container_status)" == "true" ]; then - echo "Container: collator-${{ matrix.network }}-testnet-local RUNNING"; + echo "Container: collator-selection-${{ matrix.network }} RUNNING"; echo "Check Docker logs" DOCKER_LOGS=$(do_docker_logs) if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then @@ -105,7 +203,7 @@ return 1 fi else - echo "Container collator-${{ matrix.network }}-testnet-local NOT RUNNING" + echo "Container collator-selection-${{ matrix.network }} NOT RUNNING" echo "Halting all future checks" exit 1 fi @@ -163,3 +261,4 @@ run: | docker builder prune -f -a docker system prune -f + -- gitstuff