--- /dev/null +++ b/.docker/Dockerfile-chain-dev @@ -0,0 +1,29 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=Etc/UTC + +RUN apt-get update && apt-get install -y git curl libssl-dev llvm pkg-config libclang-dev clang git make cmake + +ENV CARGO_HOME="/cargo-home" +ENV PATH="/cargo-home/bin:$PATH" + +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain none + +ARG RUST_TOOLCHAIN=nightly-2022-05-11 +ARG POLKA_VERSION=release-v0.9.24 +ARG BRANCH= +ARG REPO_URL= +ARG FEATURE= + +RUN rustup toolchain uninstall $(rustup toolchain list) && \ + rustup toolchain install $RUST_TOOLCHAIN && \ + rustup default $RUST_TOOLCHAIN && \ + rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN + +RUN mkdir /dev_chain && git clone $REPO_URL -b $BRANCH /dev_chain +WORKDIR /dev_chain + +RUN cargo build --release + +CMD cargo run --release $FEATURE -- --dev -linfo --unsafe-ws-external --rpc-cors=all --unsafe-rpc-external --- /dev/null +++ b/.docker/Dockerfile-parachain @@ -0,0 +1,84 @@ +# ===== Rust builder ===== +FROM ubuntu:20.04 as rust-builder +LABEL maintainer="Unique.Network" + +ARG RUST_TOOLCHAIN=nightly-2022-05-11 + +ENV RUST_TOOLCHAIN $RUST_TOOLCHAIN +ENV CARGO_HOME="/cargo-home" +ENV PATH="/cargo-home/bin:$PATH" + +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain none + +RUN apt-get update && \ + apt-get install -y cmake pkg-config libssl-dev git clang && \ + apt-get clean && \ + rm -r /var/lib/apt/lists/* + +RUN rustup toolchain uninstall $(rustup toolchain list) && \ + rustup toolchain install $RUST_TOOLCHAIN && \ + rustup default $RUST_TOOLCHAIN && \ + rustup target list --installed && \ + rustup show +RUN rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN + + +RUN mkdir /unique_parachain +WORKDIR /unique_parachain + +# ===== BUILD ====== +FROM rust-builder as builder-unique + +ARG PROFILE=release +ARG FEATURE= + +RUN mkdir /unique_parachain +WORKDIR /unique_parachain + +RUN git clone $REPO_URL -b $BRANCH +RUN cargo build $FEATURE --$PROFILE + +# ===== BUILD POLKADOT ===== +FROM rust-builder as builder-polkadot + +ARG POLKA_VERSION=release-v0.9.24 +ENV POLKA_VERSION $POLKA_VERSION + +RUN mkdir /unique_parachain +WORKDIR /unique_parachain + +RUN git clone -b $POLKA_VERSION --depth 1 https://github.com/paritytech/polkadot.git && \ + cd polkadot && \ + cargo build --release + +# ===== RUN ====== + +FROM ubuntu:20.04 + +RUN apt-get -y update && \ + apt-get -y upgrade && \ + apt-get -y install curl git && \ + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \ + export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + nvm install v16.16.0 && \ + nvm use v16.16.0 + +RUN git clone https://github.com/uniquenetwork/polkadot-launch -b feature/runtime-upgrade-testing + +RUN export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + cd /polkadot-launch && \ + npm install --global yarn && \ + yarn install + +COPY --from=builder-unique /unique_parachain/target/release/unique-collator /unique-chain/target/release/ +COPY --from=builder-polkadot /unique_parachain/polkadot/target/release/polkadot /polkadot/target/release/ + +COPY --from=builder-unique /unique_parachain/launch-config.json /polkadot-launch/launch-config.json + + +CMD export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + cd /polkadot-launch && \ + yarn start launch-config.json --- /dev/null +++ b/.docker/docker-compose-dev.yaml @@ -0,0 +1,19 @@ +version: "3.5" + +services: + node-dev: + build: + context: ../ + dockerfile: .docker/Dockerfile-chain-dev + image: node-dev + container_name: node-dev + expose: + - 9944 + - 9933 + ports: + - 127.0.0.1:9944:9944 + - 127.0.0.1:9933:9933 + logging: + options: + max-size: "1m" + max-file: "3" --- /dev/null +++ b/.docker/docker-compose.tmp.j2 @@ -0,0 +1,13 @@ +version: "3.5" + +services: + node-dev: + build: + args: + - "RUST_TOOLCHAIN={{ RUST_TOOLCHAIN }}" + - "BRANCH={{ BRANCH }}" + - "REPO_URL={{ REPO_URL }}" + - "FEATURE={{ FEATURE }}" + - "POLKA_VERSION={{ POLKA_VERSION }}" + + command: cargo run --release $FEATURE -- --dev -linfo --unsafe-ws-external --rpc-cors=all --unsafe-rpc-external --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .git/ .github/ +.docker/ doc/ target/ tests/ --- a/.github/workflows/codestyle.yml +++ b/.github/workflows/codestyle.yml @@ -1,12 +1,37 @@ name: Code style -on: [push] +on: + pull_request: + branches: + - develop + types: + - opened + - reopened + - synchronize jobs: rustfmt: - runs-on: ubuntu-20.04 + runs-on: self-hosted-ci steps: + - name: Skip if pull request is in Draft + # `if: github.event.pull_request.draft == true` should be kept here, at + # the step level, rather than at the job level. The latter is not + # recommended because when the PR is moved from "Draft" to "Ready to + # review" the workflow will immediately be passing (since it was skipped), + # even though it hasn't actually ran, since it takes a few seconds for + # the workflow to start. This is also disclosed in: + # https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/17 + # That scenario would open an opportunity for the check to be bypassed: + # 1. Get your PR approved + # 2. Move it to Draft + # 3. Push whatever commits you want + # 4. Move it to "Ready for review"; now the workflow is passing (it was + # skipped) and "Check reviews" is also passing (it won't be updated + # until the workflow is finished) + if: github.event.pull_request.draft == true + run: exit 1 + - uses: actions/checkout@v1 - name: Install latest nightly uses: actions-rs/toolchain@v1 @@ -20,10 +45,28 @@ clippy: if: ${{ false }} - runs-on: ubuntu-20.04 + runs-on: self-hosted-ci steps: - - uses: actions/checkout@v1 + - name: Skip if pull request is in Draft + # `if: github.event.pull_request.draft == true` should be kept here, at + # the step level, rather than at the job level. The latter is not + # recommended because when the PR is moved from "Draft" to "Ready to + # review" the workflow will immediately be passing (since it was skipped), + # even though it hasn't actually ran, since it takes a few seconds for + # the workflow to start. This is also disclosed in: + # https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/17 + # That scenario would open an opportunity for the check to be bypassed: + # 1. Get your PR approved + # 2. Move it to Draft + # 3. Push whatever commits you want + # 4. Move it to "Ready for review"; now the workflow is passing (it was + # skipped) and "Check reviews" is also passing (it won't be updated + # until the workflow is finished) + if: github.event.pull_request.draft == true + run: exit 1 + + - uses: actions/checkout@v3 - name: Install substrate dependencies run: sudo apt-get install libssl-dev pkg-config libclang-dev clang - name: Install latest nightly --- a/.github/workflows/node_build_test.yml +++ b/.github/workflows/node_build_test.yml @@ -1,44 +1,115 @@ -name: Build & test +name: Develop - Build & test -# Controls when the action will run. +# Controls when the action will run. on: # Triggers the workflow on push or pull request events but only for the master branch - push: - branches: [ develop ] - # pull_request: - # branches: [ develop ] + pull_request: + branches: + - develop + types: + - opened + - reopened + - synchronize #commit(s) pushed to the pull request + # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +#Define Workflow variables +env: + RUST_TOOLCHAIN: nightly-2022-05-11 + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + POLKA_VERSION: release-v0.9.24 + # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: build: # The type of runner that the job will run on - runs-on: ubuntu-20.04 + runs-on: self-hosted-ci + + name: Build Container, Spin it Up an test - # if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true + continue-on-error: false #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. - # Steps represent a sequence of tasks that will be executed as part of the job + strategy: + matrix: + include: + - network: "Opal" + features: "--features=opal-runtime" + - network: "Quartz" + features: "--features=quartz-runtime" + - network: "Unique" + features: "--features=unique-runtime" + steps: + - name: Skip if pull request is in Draft + # `if: github.event.pull_request.draft == true` should be kept here, at + # the step level, rather than at the job level. The latter is not + # recommended because when the PR is moved from "Draft" to "Ready to + # review" the workflow will immediately be passing (since it was skipped), + # even though it hasn't actually ran, since it takes a few seconds for + # the workflow to start. This is also disclosed in: + # https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/17 + # That scenario would open an opportunity for the check to be bypassed: + # 1. Get your PR approved + # 2. Move it to Draft + # 3. Push whatever commits you want + # 4. Move it to "Ready for review"; now the workflow is passing (it was + # skipped) and "Check reviews" is also passing (it won't be updated + # until the workflow is finished) + if: github.event.pull_request.draft == true + run: exit 1 + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - #runs ssh connection - - name: Go to server - uses: appleboy/ssh-action@master + - uses: actions/checkout@v3 with: - host: ${{ secrets.SERVER_IP }} - username: ${{ secrets.SERVER_USERNAME }} - key: ${{ secrets.KEY }} - port: ${{ secrets.SERVER_PORT }} - command_timeout: 300m - script: | - eval $(ssh-agent -s) - ssh-add /home/devops/.ssh/git_hub - git clone git@github.com:UniqueNetwork/unique-chain.git - cd unique-chain - git checkout develop - # git pull --all - chmod +x ci_node.sh - ./ci_node.sh - rm -rf /home/polkadot/unique-chain + ref: ${{ github.head_ref }} #Checking out head commit + + - name: Generate ENV related extend file for docker-compose + uses: cuchi/jinja2-action@v1.2.0 + with: + template: .docker/docker-compose.tmp.j2 + output_file: .docker/docker-compose.${{ matrix.network }}.yml + variables: | + REPO_URL=${{ github.server_url }}/${{ github.repository }}.git + RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} + POLKA_VERSION=${{ env.POLKA_VERSION }} + FEATURE='${{ matrix.features }}' + BRANCH=${{ github.head_ref }} + + - name: Show build configuration + run: cat .docker/docker-compose.${{ matrix.network }}.yml + + - name: Build the stack + run: docker-compose -f ".docker/docker-compose-dev.yaml" -f ".docker/docker-compose.${{ matrix.network }}.yml" up -d --build + + - uses: actions/setup-node@v3 + with: + node-version: 16 + + - name: Run tests + run: | + cd tests + yarn install + yarn add mochawesome + echo "Ready to start tests" + node scripts/readyness.js + NOW=$(date +%s) && yarn test --reporter mochawesome --reporter-options reportFilename=test-${NOW} + env: + RPC_URL: http://127.0.0.1:9933/ + + - name: Test Report + uses: phoenix-actions/test-reporting@v8 + if: success() || failure() # run this step even if previous step failed + with: + name: Tests ${{ matrix.network }} # Name of the check run which will be created + path: tests/mochawesome-report/test-*.json # Path to test results + reporter: mochawesome-json + fail-on-error: 'false' + + - name: Read output variables + run: | + echo "url is ${{ steps.test-report.outputs.runHtmlUrl }}" + + - name: Stop running containers + if: always() # run this step always + run: docker-compose -f ".docker/docker-compose-dev.yaml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down --- a/.github/workflows/notify.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: telegram message -on: - push: - branches: [ develop ] -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: avkviring/telegram-github-action@v0.0.13 - env: - telegram_to: ${{ secrets.TELEGRAM_TO }} - telegram_token: ${{ secrets.TELEGRAM_TOKEN }} - event: ${{ toJson(github.event) }} --- a/.github/workflows/tests_codestyle.yml +++ b/.github/workflows/tests_codestyle.yml @@ -1,14 +1,43 @@ name: Tests code style -on: [push] - +on: + pull_request: + branches: + - develop + types: + - opened + - reopened + - synchronize jobs: build: - runs-on: ubuntu-20.04 + runs-on: self-hosted-ci steps: - - uses: actions/checkout@v2 - - name: Install modules - run: cd tests && yarn - - name: Run ESLint - run: cd tests && yarn eslint --ext .ts,.js src/ \ No newline at end of file + - name: Skip if pull request is in Draft + # `if: github.event.pull_request.draft == true` should be kept here, at + # the step level, rather than at the job level. The latter is not + # recommended because when the PR is moved from "Draft" to "Ready to + # review" the workflow will immediately be passing (since it was skipped), + # even though it hasn't actually ran, since it takes a few seconds for + # the workflow to start. This is also disclosed in: + # https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/17 + # That scenario would open an opportunity for the check to be bypassed: + # 1. Get your PR approved + # 2. Move it to Draft + # 3. Push whatever commits you want + # 4. Move it to "Ready for review"; now the workflow is passing (it was + # skipped) and "Check reviews" is also passing (it won't be updated + # until the workflow is finished) + if: github.event.pull_request.draft == true + run: exit 1 + + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: 16 + + - name: Install modules + run: cd tests && yarn + - name: Run ESLint + run: cd tests && yarn eslint --ext .ts,.js src/ --- a/Dockerfile-parachain +++ /dev/null @@ -1,107 +0,0 @@ -# ===== Rust builder ===== -FROM phusion/baseimage:focal-1.1.0 as rust-builder -LABEL maintainer="Unique.Network" - -ARG RUST_TOOLCHAIN=nightly-2022-05-11 -#ARG RUST_C=1.62.0-nightly -ARG POLKA_VERSION=release-v0.9.24 -ARG UNIQUE_BRANCH=develop - -#ARG USER=*** -#ARG PASS=*** - - -ENV RUST_TOOLCHAIN $RUST_TOOLCHAIN -#ENV RUST_C $RUST_C -ENV POLKA_VERSION $POLKA_VERSION -ENV UNIQUE_BRANCH $UNIQUE_BRANCH - - -#RUN echo $RUST_TOOLCHAIN -#RUN echo $RUST_C -#RUN echo $POLKA_VERSION -#RUN echo $UNIQUE_BRANCH - -ENV CARGO_HOME="/cargo-home" -ENV PATH="/cargo-home/bin:$PATH" - -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain none - -RUN apt-get update && \ - apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ - apt-get install -y cmake pkg-config libssl-dev git clang - -RUN rustup toolchain uninstall $(rustup toolchain list) && \ - rustup toolchain install $RUST_TOOLCHAIN && \ - rustup default $RUST_TOOLCHAIN && \ - rustup target list --installed && \ - rustup show -RUN rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN - -RUN cargo install cargo-chef - -RUN mkdir unique_parachain -WORKDIR /unique_parachain - -# ===== Chef ===== -FROM rust-builder as chef - -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - -# ===== BUILD ====== -FROM rust-builder as builder - -RUN mkdir unique_parachain -WORKDIR /unique_parachain - -COPY --from=chef /unique_parachain/recipe.json recipe.json -ARG PROFILE=release -RUN cargo chef cook "--$PROFILE" --recipe-path recipe.json - -COPY . . -RUN cargo build "--$PROFILE" - # && \ - # cargo test - -# ===== BUILD POLKADOT ===== -FROM rust-builder as builder-polkadot - -RUN mkdir unique_parachain -WORKDIR /unique_parachain - -RUN git clone -b $POLKA_VERSION --depth 1 https://github.com/paritytech/polkadot.git && \ - cd polkadot && \ - git tag -n && \ - cargo build --release - -# ===== RUN ====== - -FROM phusion/baseimage:focal-1.1.0 - -ARG PROFILE=release - -RUN apt-get -y update && \ - apt-get -y upgrade && \ - apt-get -y install curl git && \ - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \ - export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - nvm install v15.5.0 && \ - nvm use v15.5.0 - -RUN git clone https://github.com/paritytech/polkadot-launch - -RUN export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - cd /polkadot-launch && \ - npm install --global yarn && \ - yarn - -COPY --from=builder /unique_parachain/target/$PROFILE/unique-collator /unique-chain/target/$PROFILE/ -COPY --from=builder-polkadot /unique_parachain/polkadot/target/$PROFILE/polkadot /polkadot/target/$PROFILE/ - -CMD export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - cd /polkadot-launch && \ - yarn start launch-config.json --- a/docker-compose-tests-parachain.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: "3.5" - -services: - blockchain_nodes: - build: - context: ./ - dockerfile: Dockerfile-parachain - args: - - RUST_TOOLCHAIN=${RUST_TOOLCHAIN:?err} - - RUST_C=${RUST_C:?err} - - POLKA_VERSION=${POLKA_VERSION:?err} - - UNIQUE_BRANCH=${UNIQUE_BRANCH:?err} - volumes: - - ./launch-config.json:/polkadot-launch/launch-config.json - env_file: - - ./.env - - integration_tests: - build: - context: tests/ - dockerfile: Dockerfile-tests - environment: - RPC_URL: http://blockchain_nodes:9933/ - volumes: - - ./tests/src/config_docker.ts:/unique_parachain/src/config.ts - - /home/ubuntu/mochawesome-report:/unique_parachain/mochawesome-report - depends_on: - - blockchain_nodes --- /dev/null +++ b/tests/scripts/readyness.js @@ -0,0 +1,35 @@ +const { ApiPromise, WsProvider } = require('@polkadot/api'); + +const connect = async () => { + const wsEndpoint = 'ws://127.0.0.1:9944' + const api = new ApiPromise({provider: new WsProvider(wsEndpoint)}); + await api.isReadyOrError; + + const head = (await api.rpc.chain.getHeader()).number.toNumber(); + if(head < 1) throw Error('No block #1'); + + await api.disconnect(); +} + +const sleep = time => { + return new Promise(resolve => { + setTimeout(() => resolve(), time); + }); +} + +const main = async () => { + while(true) { + try { + await connect(); + break; + } + catch(e) { + await sleep(10000); + } + } +} + +main().then(() => process.exit(0)).catch(e => { + console.error(e); + process.exit(1); +});