From 6fd78602200ef1ec38d91ca2569f9563a1d501aa Mon Sep 17 00:00:00 2001 From: Alexander Aksenov Date: Thu, 18 Aug 2022 15:14:23 +0000 Subject: [PATCH] tests: change docker-compose file and workflow for forkless upgrade --- --- a/.docker/Dockerfile-parachain-upgrade +++ b/.docker/Dockerfile-parachain-upgrade @@ -25,8 +25,8 @@ RUN rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN -#RUN mkdir /unique_parachain -#WORKDIR /unique_parachain +RUN mkdir /unique_parachain +WORKDIR /unique_parachain # ===== BUILD current version ====== @@ -37,7 +37,6 @@ ARG MAINNET_BRANCH= ARG REPO_URL= -RUN mkdir /unique_parachain WORKDIR /unique_parachain RUN git clone $REPO_URL -b $MAINNET_BRANCH . && \ @@ -52,11 +51,10 @@ ARG BRANCH= ARG REPO_URL= -RUN mkdir /unique_parachain +COPY . /unique_parachain WORKDIR /unique_parachain -RUN git clone $REPO_URL -b $BRANCH . && \ - cargo build --features=$FEATURE --$PROFILE +RUN cargo build --features=$FEATURE --$PROFILE # ===== BUILD POLKADOT ===== FROM rust-builder as builder-polkadot @@ -64,7 +62,7 @@ ARG POLKADOT_BUILD_BRANCH= ENV POLKADOT_BUILD_BRANCH $POLKADOT_BUILD_BRANCH -RUN mkdir /unique_parachain + WORKDIR /unique_parachain RUN git clone -b $POLKADOT_BUILD_BRANCH --depth 1 https://github.com/paritytech/polkadot.git && \ --- /dev/null +++ b/.docker/Dockerfile-parachain-upgrade-data @@ -0,0 +1,103 @@ +# ===== Rust builder ===== +FROM ubuntu:20.04 as rust-builder +LABEL maintainer="Unique.Network" + +ARG RUST_TOOLCHAIN= + +ENV RUST_TOOLCHAIN $RUST_TOOLCHAIN + +ENV CARGO_HOME="/cargo-home" +ENV PATH="/cargo-home/bin:$PATH" +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update && \ + apt-get install -y curl cmake pkg-config libssl-dev git clang && \ + apt-get clean && \ + rm -r /var/lib/apt/lists/* + +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain none + +RUN rustup toolchain uninstall $(rustup toolchain list) && \ + rustup toolchain install $RUST_TOOLCHAIN && \ + rustup default $RUST_TOOLCHAIN && \ + rustup target list --installed && \ + rustup show +RUN rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN + +RUN mkdir /unique_parachain +WORKDIR /unique_parachain + +# ===== BUILD target version ====== +FROM rust-builder as builder-unique-target + +ARG PROFILE=release +ARG FEATURE= +ARG BRANCH= +ARG REPO_URL= + +COPY . /unique_parachain +WORKDIR /unique_parachain + +RUN cargo build --features=$FEATURE --$PROFILE + +# ===== BUILD POLKADOT ===== +FROM rust-builder as builder-polkadot + +ARG POLKADOT_BUILD_BRANCH= +ENV POLKADOT_BUILD_BRANCH $POLKADOT_BUILD_BRANCH + +WORKDIR /unique_parachain + +RUN git clone -b $POLKADOT_BUILD_BRANCH --depth 1 https://github.com/paritytech/polkadot.git && \ + cd polkadot && \ + cargo build --release + +# ===== BUILD CHAINQL ===== +FROM rust-builder as builder-chainql + +RUN mkdir chainql +WORKDIR /chainql + +RUN git clone --depth 1 https://github.com/CertainLach/chainql.git . && \ + cargo build --release + +# ===== RUN ====== + +FROM ubuntu:20.04 + +ARG RUNTIME= +ENV RUNTIME $RUNTIME + +RUN apt-get -y update && \ + apt-get -y install curl git && \ + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \ + export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + nvm install v16.16.0 && \ + nvm use v16.16.0 + +RUN git clone https://github.com/uniquenetwork/polkadot-launch -b feature/parachain-forking + +RUN export NVM_DIR="$HOME/.nvm" && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + cd /polkadot-launch && \ + npm install --global yarn && \ + yarn install + +RUN echo "$RUNTIME" + +COPY --from=builder-unique-target /unique_parachain/target/release/unique-collator /unique-chain/target/release/ + +COPY --from=builder-polkadot /unique_parachain/polkadot/target/release/polkadot /polkadot/target/release/ + +COPY --from=builder-chainql /chainql/target/release/chainql /chainql/target/release/ + +ARG FORK_FROM= +ENV FORK_FROM=$FORK_FROM +CMD export NVM_DIR="$HOME/.nvm" PATH="$PATH:/chainql/target/release" FORK_FROM && \ + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ + cd /polkadot-launch && \ + yarn start launch-config.json + + --- a/.docker/docker-compose-forkless.yaml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3.5" - -services: - node-parachain: - build: - context: ../ - dockerfile: .docker/Dockerfile-parachain-upgrade - image: node-parachain - container_name: node-parachain - volumes: - - type: bind - source: ./launch-config-forkless.json - target: /polkadot-launch/launch-config.json - read_only: true - expose: - - 9944 - - 9933 - ports: - - 127.0.0.1:9944:9944 - - 127.0.0.1:9933:9933 - logging: - options: - max-size: "1m" - max-file: "3" --- /dev/null +++ b/.docker/docker-compose-forkless.yml @@ -0,0 +1,24 @@ +version: "3.5" + +services: + node-parachain: + build: + context: ../ + dockerfile: .docker/Dockerfile-parachain-upgrade + image: node-parachain + container_name: node-parachain + volumes: + - type: bind + source: ./launch-config-forkless.json + target: /polkadot-launch/launch-config.json + read_only: true + expose: + - 9944 + - 9933 + ports: + - 127.0.0.1:9944:9944 + - 127.0.0.1:9933:9933 + logging: + options: + max-size: "1m" + max-file: "3" --- /dev/null +++ b/.docker/docker-compose.opal.yml @@ -0,0 +1,33 @@ +version: "3.5" + +services: + node-parachain: + build: + args: + - "RUST_TOOLCHAIN=nightly-2022-07-24" + - "BRANCH=release-v927020" + - "REPO_URL=https://github.com/UniqueNetwork/unique-chain.git" + - "FEATURE=quartz-runtime" + - "RUNTIME=quartz" + - "POLKADOT_BUILD_BRANCH=release-v0.9.27" + - "MAINNET_TAG=" + - "MAINNET_BRANCH=quartz-v924012-2" + context: ../ + dockerfile: .docker/Dockerfile-parachain-upgrade + image: node-parachain + container_name: node-parachain + volumes: + - type: bind + source: ./launch-config-forkless-nodata.json + target: /polkadot-launch/launch-config.json + read_only: true + expose: + - 9944 + - 9933 + ports: + - 127.0.0.1:9944:9944 + - 127.0.0.1:9933:9933 + logging: + options: + max-size: "1m" + max-file: "3" --- /dev/null +++ b/.docker/docker-compose.tmp-forkless-data.j2 @@ -0,0 +1,36 @@ +version: "3.5" + +services: + node-parachain: + build: + args: + - "RUST_TOOLCHAIN={{ RUST_TOOLCHAIN }}" + - "BRANCH={{ BRANCH }}" + - "REPO_URL={{ REPO_URL }}" + - "FEATURE={{ FEATURE }}" + - "RUNTIME={{ RUNTIME }}" + - "POLKADOT_BUILD_BRANCH={{ POLKADOT_BUILD_BRANCH }}" + - "FORK_FROM={{ FORK_FROM }}" + context: ./ + dockerfile: ./Dockerfile-parachain-upgrade-data + image: node-parachain + container_name: node-parachain + volumes: + - type: bind + source: ./launch-config-forkless-data.json + target: /polkadot-launch/launch-config.json + read_only: true + - type: bind + source: ./fork.jsonnet + target: /polkadot-launch/fork.jsonnet + read_only: true + expose: + - 9944 + - 9933 + ports: + - 127.0.0.1:9944:9944 + - 127.0.0.1:9933:9933 + logging: + options: + max-size: "1m" + max-file: "3" \ No newline at end of file --- /dev/null +++ b/.docker/docker-compose.tmp-forkless-nodata.j2 @@ -0,0 +1,33 @@ +version: "3.5" + +services: + node-parachain: + build: + args: + - "RUST_TOOLCHAIN={{ RUST_TOOLCHAIN }}" + - "BRANCH={{ BRANCH }}" + - "REPO_URL={{ REPO_URL }}" + - "FEATURE={{ FEATURE }}" + - "RUNTIME={{ RUNTIME }}" + - "POLKADOT_BUILD_BRANCH={{ POLKADOT_BUILD_BRANCH }}" + - "MAINNET_TAG={{ MAINNET_TAG }}" + - "MAINNET_BRANCH={{ MAINNET_BRANCH }}" + context: ../ + dockerfile: .docker/Dockerfile-parachain-upgrade + image: node-parachain + container_name: node-parachain + volumes: + - type: bind + source: ./launch-config-forkless-nodata.json + target: /polkadot-launch/launch-config.json + read_only: true + expose: + - 9944 + - 9933 + ports: + - 127.0.0.1:9944:9944 + - 127.0.0.1:9933:9933 + logging: + options: + max-size: "1m" + max-file: "3" --- a/.docker/docker-compose.tmp-forkless.j2 +++ /dev/null @@ -1,14 +0,0 @@ -version: "3.5" - -services: - node-parachain: - build: - args: - - "RUST_TOOLCHAIN={{ RUST_TOOLCHAIN }}" - - "BRANCH={{ BRANCH }}" - - "REPO_URL={{ REPO_URL }}" - - "FEATURE={{ FEATURE }}" - - "RUNTIME={{ RUNTIME }}" - - "POLKADOT_BUILD_BRANCH={{ POLKADOT_BUILD_BRANCH }}" - - "MAINNET_TAG={{ MAINNET_TAG }}" - - "MAINNET_BRANCH={{ MAINNET_BRANCH }}" --- a/.docker/forking/Dockerfile-parachain-live-fork +++ /dev/null @@ -1,102 +0,0 @@ -# ===== Rust builder ===== -FROM ubuntu:20.04 as rust-builder -LABEL maintainer="Unique.Network" - -ARG RUST_TOOLCHAIN= - -ENV RUST_TOOLCHAIN $RUST_TOOLCHAIN - -ENV CARGO_HOME="/cargo-home" -ENV PATH="/cargo-home/bin:$PATH" -ENV TZ=UTC -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN apt-get update && \ - apt-get install -y curl cmake pkg-config libssl-dev git clang && \ - apt-get clean && \ - rm -r /var/lib/apt/lists/* - -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain none - -RUN rustup toolchain uninstall $(rustup toolchain list) && \ - rustup toolchain install $RUST_TOOLCHAIN && \ - rustup default $RUST_TOOLCHAIN && \ - rustup target list --installed && \ - rustup show -RUN rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN - -# ===== BUILD target version ====== -FROM rust-builder as builder-unique-target - -ARG PROFILE=release -ARG FEATURE= -ARG BRANCH= -ARG REPO_URL= - -RUN mkdir /unique_parachain -WORKDIR /unique_parachain - -RUN git clone $REPO_URL -b $BRANCH . && \ - cargo build --features=$FEATURE --$PROFILE - -# ===== BUILD POLKADOT ===== -FROM rust-builder as builder-polkadot - -ARG POLKADOT_BUILD_BRANCH= -ENV POLKADOT_BUILD_BRANCH $POLKADOT_BUILD_BRANCH - -RUN mkdir /unique_parachain -WORKDIR /unique_parachain - -RUN git clone -b $POLKADOT_BUILD_BRANCH --depth 1 https://github.com/paritytech/polkadot.git && \ - cd polkadot && \ - cargo build --release - -# ===== BUILD CHAINQL ===== -FROM rust-builder as builder-chainql - -RUN mkdir chainql -WORKDIR /chainql - -RUN git clone --depth 1 https://github.com/CertainLach/chainql.git . && \ - cargo build --release - -# ===== RUN ====== - -FROM ubuntu:20.04 - -ARG RUNTIME= -ENV RUNTIME $RUNTIME - -RUN apt-get -y update && \ - apt-get -y install curl git && \ - curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \ - export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - nvm install v16.16.0 && \ - nvm use v16.16.0 - -RUN git clone https://github.com/uniquenetwork/polkadot-launch -b feature/parachain-forking - -RUN export NVM_DIR="$HOME/.nvm" && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - cd /polkadot-launch && \ - npm install --global yarn && \ - yarn install - -RUN echo "$RUNTIME" - -COPY --from=builder-unique-target /unique_parachain/target/release/unique-collator /unique-chain/target/release/ - -COPY --from=builder-polkadot /unique_parachain/polkadot/target/release/polkadot /polkadot/target/release/ - -COPY --from=builder-chainql /chainql/target/release/chainql /chainql/target/release/ - -ARG FORK_FROM= -ENV FORK_FROM=$FORK_FROM -CMD export NVM_DIR="$HOME/.nvm" PATH="$PATH:/chainql/target/release" FORK_FROM && \ - [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ - cd /polkadot-launch && \ - yarn start launch-config.json - - --- a/.docker/forking/docker-compose-fork.yaml +++ /dev/null @@ -1,28 +0,0 @@ -version: "3.5" - -services: - parachain-fork: - build: - context: ./ - dockerfile: ./Dockerfile-parachain-live-fork - image: parachain-fork - container_name: parachain-fork - volumes: - - type: bind - source: ./launch-config-fork.json - target: /polkadot-launch/launch-config.json - read_only: true - - type: bind - source: ./fork.jsonnet - target: /polkadot-launch/fork.jsonnet - read_only: true - expose: - - 9944 - - 9933 - ports: - - 127.0.0.1:9944:9944 - - 127.0.0.1:9933:9933 - logging: - options: - max-size: "1m" - max-file: "3" --- a/.docker/forking/docker-compose.tmp-fork.j2 +++ /dev/null @@ -1,13 +0,0 @@ -version: "3.5" - -services: - parachain-fork: - build: - args: - - "RUST_TOOLCHAIN={{ RUST_TOOLCHAIN }}" - - "BRANCH={{ BRANCH }}" - - "REPO_URL={{ REPO_URL }}" - - "FEATURE={{ FEATURE }}" - - "RUNTIME={{ RUNTIME }}" - - "POLKADOT_BUILD_BRANCH={{ POLKADOT_BUILD_BRANCH }}" - - "FORK_FROM={{ FORK_FROM }}" --- a/.docker/forking/fork.jsonnet +++ /dev/null @@ -1,59 +0,0 @@ - -function(rawSpec, forkFrom) -local sourceChain = cql.chain(forkFrom).latest; - -local raw = local sourceRaw = sourceChain._raw._preloadKeys; { - [key]: cql.toHex(sourceRaw[key]) - for key in std.objectFields(sourceRaw) - if sourceRaw[key] != null -}; - -local -auraKeys = [ - // AuraExt.Authorities, we don't have aura pallet enabled for some reason, to refer using cql api - '0x3c311d57d4daf52904616cf69648081e5e0621c4869aa60c02be9adcc98a0d1d', - // Aura.Authorities - '0x57f8dc2f5ab09467896f47300f0424385e0621c4869aa60c02be9adcc98a0d1d', -], - -// Keys, which should be migrated from passed spec, rather than from forked chain -wantedKeys = [ - sourceChain.ParachainInfo._key.ParachainId, - sourceChain.Sudo._key.Key, - sourceChain.System.BlockHash._key['0'], - sourceChain.System._key.ParentHash, -] + auraKeys, - -// Keys to remove from original chain -unwantedPrefixes = [ - // Aura.CurrentSlot - '0x57f8dc2f5ab09467896f47300f04243806155b3cd9a8c9e5e9a23fd5dc13a5ed', - // Ensure there will be no panics caused by unexpected kept state - sourceChain.ParachainSystem._key.ValidationData, - sourceChain.ParachainSystem._key.RelayStateProof, - sourceChain.ParachainSystem._key.HostConfiguration, - sourceChain.ParachainSystem._key.LastDmqMqcHead, - // Part of head - sourceChain.System._key.BlockHash, - sourceChain.System._key.Number, - sourceChain.System._key.Digest, -] + auraKeys, - -cleanupRaw(raw) = { - [key]: raw[key] - for key in std.objectFields(raw) - if std.all(std.map(function(prefix) !std.startsWith(key, prefix), unwantedPrefixes)) -}; - -local originalRaw = rawSpec.genesis.raw.top; -local outSpec = rawSpec { - genesis+: { - raw+: { - top: cleanupRaw(raw) { - [key]: originalRaw[key] - for key in wantedKeys - }, - }, - }, -}; -outSpec --- a/.docker/forking/launch-config-fork.j2 +++ /dev/null @@ -1,128 +0,0 @@ -{ - "relaychain": { - "bin": "/polkadot/target/release/polkadot", - "chain": "westend-local", - "nodes": [ - { - "name": "alice", - "wsPort": 9844, - "rpcPort": 9843, - "port": 30444, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - }, - { - "name": "bob", - "wsPort": 9855, - "rpcPort": 9854, - "port": 30555, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - }, - { - "name": "charlie", - "wsPort": 9866, - "rpcPort": 9865, - "port": 30666, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - }, - { - "name": "dave", - "wsPort": 9877, - "rpcPort": 9876, - "port": 30777, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - }, - { - "name": "eve", - "wsPort": 9888, - "rpcPort": 9887, - "port": 30888, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - } - ], - "genesis": { - "runtime": { - "runtime_genesis_config": { - "parachainsConfiguration": { - "config": { - "validation_upgrade_frequency": 1, - "validation_upgrade_delay": 1 - } - } - } - } - } - }, - "parachains": [ - { - "bin": "/unique-chain/target/release/unique-collator", - "id": "1000", - "balance": "1000000000000000000000000", - "chainRawInitializer": [ - "chainql", - "--ext-str=FORK_FROM", - "--tla-code=rawSpec=import '${rawSpec}'", - "--tla-code=forkFrom=std.extVar('FORK_FROM')", - "fork.jsonnet" - ], - "nodes": [ - { - "port": 31200, - "wsPort": 9944, - "rpcPort": 9933, - "name": "alice", - "flags": [ - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external", - "-lxcm=trace" - ] - }, - { - "port": 31201, - "wsPort": 9945, - "rpcPort": 9934, - "name": "bob", - "flags": [ - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external", - "-lxcm=trace" - ] - } - ] - } - ], - "simpleParachains": [], - "hrmpChannels": [], - "finalization": false -} --- /dev/null +++ b/.docker/forkless-config/fork.jsonnet @@ -0,0 +1,59 @@ + +function(rawSpec, forkFrom) +local sourceChain = cql.chain(forkFrom).latest; + +local raw = local sourceRaw = sourceChain._raw._preloadKeys; { + [key]: cql.toHex(sourceRaw[key]) + for key in std.objectFields(sourceRaw) + if sourceRaw[key] != null +}; + +local +auraKeys = [ + // AuraExt.Authorities, we don't have aura pallet enabled for some reason, to refer using cql api + '0x3c311d57d4daf52904616cf69648081e5e0621c4869aa60c02be9adcc98a0d1d', + // Aura.Authorities + '0x57f8dc2f5ab09467896f47300f0424385e0621c4869aa60c02be9adcc98a0d1d', +], + +// Keys, which should be migrated from passed spec, rather than from forked chain +wantedKeys = [ + sourceChain.ParachainInfo._key.ParachainId, + sourceChain.Sudo._key.Key, + sourceChain.System.BlockHash._key['0'], + sourceChain.System._key.ParentHash, +] + auraKeys, + +// Keys to remove from original chain +unwantedPrefixes = [ + // Aura.CurrentSlot + '0x57f8dc2f5ab09467896f47300f04243806155b3cd9a8c9e5e9a23fd5dc13a5ed', + // Ensure there will be no panics caused by unexpected kept state + sourceChain.ParachainSystem._key.ValidationData, + sourceChain.ParachainSystem._key.RelayStateProof, + sourceChain.ParachainSystem._key.HostConfiguration, + sourceChain.ParachainSystem._key.LastDmqMqcHead, + // Part of head + sourceChain.System._key.BlockHash, + sourceChain.System._key.Number, + sourceChain.System._key.Digest, +] + auraKeys, + +cleanupRaw(raw) = { + [key]: raw[key] + for key in std.objectFields(raw) + if std.all(std.map(function(prefix) !std.startsWith(key, prefix), unwantedPrefixes)) +}; + +local originalRaw = rawSpec.genesis.raw.top; +local outSpec = rawSpec { + genesis+: { + raw+: { + top: cleanupRaw(raw) { + [key]: originalRaw[key] + for key in wantedKeys + }, + }, + }, +}; +outSpec --- /dev/null +++ b/.docker/forkless-config/launch-config-forkless-data.j2 @@ -0,0 +1,128 @@ +{ + "relaychain": { + "bin": "/polkadot/target/release/polkadot", + "chain": "westend-local", + "nodes": [ + { + "name": "alice", + "wsPort": 9844, + "rpcPort": 9843, + "port": 30444, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + }, + { + "name": "bob", + "wsPort": 9855, + "rpcPort": 9854, + "port": 30555, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + }, + { + "name": "charlie", + "wsPort": 9866, + "rpcPort": 9865, + "port": 30666, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + }, + { + "name": "dave", + "wsPort": 9877, + "rpcPort": 9876, + "port": 30777, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + }, + { + "name": "eve", + "wsPort": 9888, + "rpcPort": 9887, + "port": 30888, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + } + ], + "genesis": { + "runtime": { + "runtime_genesis_config": { + "parachainsConfiguration": { + "config": { + "validation_upgrade_frequency": 1, + "validation_upgrade_delay": 1 + } + } + } + } + } + }, + "parachains": [ + { + "bin": "/unique-chain/target/release/unique-collator", + "id": "1000", + "balance": "1000000000000000000000000", + "chainRawInitializer": [ + "chainql", + "--ext-str=FORK_FROM", + "--tla-code=rawSpec=import '${rawSpec}'", + "--tla-code=forkFrom=std.extVar('FORK_FROM')", + "fork.jsonnet" + ], + "nodes": [ + { + "port": 31200, + "wsPort": 9944, + "rpcPort": 9933, + "name": "alice", + "flags": [ + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external", + "-lxcm=trace" + ] + }, + { + "port": 31201, + "wsPort": 9945, + "rpcPort": 9934, + "name": "bob", + "flags": [ + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external", + "-lxcm=trace" + ] + } + ] + } + ], + "simpleParachains": [], + "hrmpChannels": [], + "finalization": false +} --- /dev/null +++ b/.docker/forkless-config/launch-config-forkless-nodata.j2 @@ -0,0 +1,125 @@ +{ + "relaychain": { + "bin": "/polkadot/target/release/polkadot", + "upgradeBin": "/polkadot/target/release/polkadot", + "upgradeWasm": "/polkadot/target/release/wbuild/westend-runtime/westend_runtime.compact.compressed.wasm", + "chain": "westend-local", + "nodes": [ + { + "name": "alice", + "wsPort": 9844, + "rpcPort": 9843, + "port": 30444, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + }, + { + "name": "bob", + "wsPort": 9855, + "rpcPort": 9854, + "port": 30555, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + }, + { + "name": "charlie", + "wsPort": 9866, + "rpcPort": 9865, + "port": 30666, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + }, + { + "name": "dave", + "wsPort": 9877, + "rpcPort": 9876, + "port": 30777, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + }, + { + "name": "eve", + "wsPort": 9888, + "rpcPort": 9887, + "port": 30888, + "flags": [ + "-lparachain::candidate_validation=debug", + "-lxcm=trace", + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external" + ] + } + ], + "genesis": { + "runtime": { + "runtime_genesis_config": { + "parachainsConfiguration": { + "config": { + "validation_upgrade_frequency": 1, + "validation_upgrade_delay": 1 + } + } + } + } + } + }, + "parachains": [ + { + "bin": "/unique-chain/current/release/unique-collator", + "upgradeBin": "/unique-chain/target/release/unique-collator", + "upgradeWasm": "/unique-chain/target/release/wbuild/{{ FEATURE }}/{{ RUNTIME }}_runtime.compact.compressed.wasm", + "id": "1000", + "balance": "1000000000000000000000000", + "nodes": [ + { + "port": 31200, + "wsPort": 9944, + "rpcPort": 9933, + "name": "alice", + "flags": [ + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external", + "-lxcm=trace" + ] + }, + { + "port": 31201, + "wsPort": 9945, + "rpcPort": 9934, + "name": "bob", + "flags": [ + "--rpc-cors=all", + "--unsafe-rpc-external", + "--unsafe-ws-external", + "-lxcm=trace" + ] + } + ] + } + ], + "simpleParachains": [], + "hrmpChannels": [], + "finalization": false +} --- a/.docker/launch-config.j2 +++ /dev/null @@ -1,125 +0,0 @@ -{ - "relaychain": { - "bin": "/polkadot/target/release/polkadot", - "upgradeBin": "/polkadot/target/release/polkadot", - "upgradeWasm": "/polkadot/target/release/wbuild/westend-runtime/westend_runtime.compact.compressed.wasm", - "chain": "westend-local", - "nodes": [ - { - "name": "alice", - "wsPort": 9844, - "rpcPort": 9843, - "port": 30444, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - }, - { - "name": "bob", - "wsPort": 9855, - "rpcPort": 9854, - "port": 30555, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - }, - { - "name": "charlie", - "wsPort": 9866, - "rpcPort": 9865, - "port": 30666, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - }, - { - "name": "dave", - "wsPort": 9877, - "rpcPort": 9876, - "port": 30777, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - }, - { - "name": "eve", - "wsPort": 9888, - "rpcPort": 9887, - "port": 30888, - "flags": [ - "-lparachain::candidate_validation=debug", - "-lxcm=trace", - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external" - ] - } - ], - "genesis": { - "runtime": { - "runtime_genesis_config": { - "parachainsConfiguration": { - "config": { - "validation_upgrade_frequency": 1, - "validation_upgrade_delay": 1 - } - } - } - } - } - }, - "parachains": [ - { - "bin": "/unique-chain/current/release/unique-collator", - "upgradeBin": "/unique-chain/target/release/unique-collator", - "upgradeWasm": "/unique-chain/target/release/wbuild/{{ FEATURE }}/{{ RUNTIME }}_runtime.compact.compressed.wasm", - "id": "1000", - "balance": "1000000000000000000000000", - "nodes": [ - { - "port": 31200, - "wsPort": 9944, - "rpcPort": 9933, - "name": "alice", - "flags": [ - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external", - "-lxcm=trace" - ] - }, - { - "port": 31201, - "wsPort": 9945, - "rpcPort": 9934, - "name": "bob", - "flags": [ - "--rpc-cors=all", - "--unsafe-rpc-external", - "--unsafe-ws-external", - "-lxcm=trace" - ] - } - ] - } - ], - "simpleParachains": [], - "hrmpChannels": [], - "finalization": false -} --- a/.github/workflows/fork-update-withdata.yml +++ /dev/null @@ -1,170 +0,0 @@ -name: Upgrade replica - -# Controls when the action will run. -on: - # Triggers the workflow on push or pull request events but only for the master branch - pull_request: - branches: - - master - types: - - opened - - reopened - - synchronize #commit(s) pushed to the pull request - - ready_for_review - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -#Define Workflow variables -env: - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - - fork-update-withdata: - # The type of runner that the job will run on - runs-on: [self-hosted-ci,large] - timeout-minutes: 1380 - - name: ${{ matrix.network }} - - continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. - - strategy: - matrix: - include: - - network: Opal - features: opal-runtime - runtime: opal - fork_from_address: wss://eu-ws-opal.unique.network:443 - - network: Quartz - features: quartz-runtime - runtime: quartz - fork_from_address: wss://eu-ws-quartz.unique.network:443 - - network: Unique - features: unique-runtime - runtime: unique - fork_from_address: wss://eu-ws.unique.network:443 - - steps: - - name: Skip if pull request is in Draft - # `if: github.event.pull_request.draft == true` should be kept here, at - # the step level, rather than at the job level. The latter is not - # recommended because when the PR is moved from "Draft" to "Ready to - # review" the workflow will immediately be passing (since it was skipped), - # even though it hasn't actually ran, since it takes a few seconds for - # the workflow to start. This is also disclosed in: - # https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/17 - # That scenario would open an opportunity for the check to be bypassed: - # 1. Get your PR approved - # 2. Move it to Draft - # 3. Push whatever commits you want - # 4. Move it to "Ready for review"; now the workflow is passing (it was - # skipped) and "Check reviews" is also passing (it won't be updated - # until the workflow is finished) - if: github.event.pull_request.draft == true - run: exit 1 - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 - - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} #Checking out head commit - - - name: Read .env file - uses: xom9ikk/dotenv@v1.0.2 - - - name: Generate ENV related extend file for docker-compose - uses: cuchi/jinja2-action@v1.2.0 - with: - template: .docker/forking/docker-compose.tmp-fork.j2 - output_file: .docker/forking/docker-compose.${{ matrix.network }}.yml - variables: | - REPO_URL=${{ github.server_url }}/${{ github.repository }}.git - RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} - POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }} - POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }} - FEATURE=${{ matrix.features }} - RUNTIME=${{ matrix.runtime }} - BRANCH=${{ github.head_ref }} - FORK_FROM=${{ matrix.fork_from_address }} - - - name: Show build configuration - run: cat .docker/forking/docker-compose.${{ matrix.network }}.yml - - - name: Generate launch-config-fork.json - uses: cuchi/jinja2-action@v1.2.0 - with: - template: .docker/forking/launch-config-fork.j2 - output_file: .docker/forking/launch-config-fork.json - variables: | - FEATURE=${{ matrix.features }} - RUNTIME=${{ matrix.runtime }} - - - name: Show launch-config-fork configuration - run: cat .docker/forking/launch-config-fork.json - - - - name: Build the stack - run: docker-compose -f ".docker/forking/docker-compose-fork.yaml" -f ".docker/forking/docker-compose.${{ matrix.network }}.yml" up -d --build --force-recreate --timeout 300 - - - name: Check if docker logs consist logs related to Runtime Upgrade testing. - if: success() - run: | - counter=160 - function do_docker_logs { - docker logs --details parachain-fork 2>&1 - } - function is_started { - echo "Check Docker logs" - DOCKER_LOGS=$(do_docker_logs) - if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then - echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸" - return 0 - exit 0 - fi - echo "Function is_started: Return 1" - return 1 - } - while ! is_started; do - echo "Waiting for special message in log files " - sleep 30s - counter=$(( $counter - 1 )) - echo "Counter: $counter" - if [ "$counter" -gt "0" ]; then - continue - else - break - fi - done - exit 1 - shell: bash - - - name: Collect Docker Logs - if: success() || failure() - uses: jwalton/gh-docker-logs@v2.2.0 - with: - dest: './fork-parachain-update-withdata-logs.${{ matrix.features }}' - images: 'parachain-fork' - - - name: Tar logs - if: success() || failure() - run: tar cvzf ./fork-parachain-update-withdata-logs.${{ matrix.features }}.tgz ./fork-parachain-update-withdata-logs.${{ matrix.features }} - - - name: Upload logs to GitHub - if: success() || failure() - uses: actions/upload-artifact@master - with: - name: fork-parachain-update-withdata-logs.${{ matrix.features }}.tgz - path: ./fork-parachain-update-withdata-logs.${{ matrix.features }}.tgz - - - name: Stop running containers - if: always() # run this step always - run: docker-compose -f ".docker/forking/docker-compose-fork.yaml" -f ".docker/forking/docker-compose.${{ matrix.network }}.yml" down --- /dev/null +++ b/.github/workflows/forkless-update-data.yml @@ -0,0 +1,162 @@ +name: upgrade replica + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the master branch + pull_request: + branches: + - master + types: + - opened + - reopened + - synchronize #commit(s) pushed to the pull request + - ready_for_review + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +#Define Workflow variables +env: + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + + forkless-update-data: + # The type of runner that the job will run on + runs-on: [self-hosted-ci,large] + timeout-minutes: 1380 + + name: ${{ matrix.network }} + + continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. + + strategy: + matrix: + include: + - network: opal + features: opal-runtime + runtime: opal + fork_from_address: wss://eu-ws-opal.unique.network:443 + - network: quartz + features: quartz-runtime + runtime: quartz + fork_from_address: wss://eu-ws-quartz.unique.network:443 + - network: unique + features: unique-runtime + runtime: unique + fork_from_address: wss://eu-ws.unique.network:443 + + steps: + - name: Skip if pull request is in Draft + # `if: github.event.pull_request.draft == true` should be kept here, at + # the step level, rather than at the job level. The latter is not + # recommended because when the PR is moved from "Draft" to "Ready to + # review" the workflow will immediately be passing (since it was skipped), + # even though it hasn't actually ran, since it takes a few seconds for + # the workflow to start. This is also disclosed in: + # https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/17 + # That scenario would open an opportunity for the check to be bypassed: + # 1. Get your PR approved + # 2. Move it to Draft + # 3. Push whatever commits you want + # 4. Move it to "Ready for review"; now the workflow is passing (it was + # skipped) and "Check reviews" is also passing (it won't be updated + # until the workflow is finished) + if: github.event.pull_request.draft == true + run: exit 1 + + - name: Clean Workspace + uses: AutoModality/action-clean@v1.1.0 + + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} #Checking out head commit + + - name: Read .env file + uses: xom9ikk/dotenv@v1.0.2 + + - name: Generate ENV related extend file for docker-compose + uses: cuchi/jinja2-action@v1.2.0 + with: + template: .docker/docker-compose.tmp-forkless-data.j2 + output_file: .docker/docker-compose.${{ matrix.network }}.yml + variables: | + REPO_URL=${{ github.server_url }}/${{ github.repository }}.git + RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} + POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }} + POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }} + FEATURE=${{ matrix.features }} + RUNTIME=${{ matrix.runtime }} + BRANCH=${{ github.head_ref }} + FORK_FROM=${{ matrix.fork_from_address }} + + - name: Show build configuration + run: cat .docker/forking/docker-compose.${{ matrix.network }}.yml + + - name: Generate launch-config-forkless-data.json + uses: cuchi/jinja2-action@v1.2.0 + with: + template: .docker/forkless-config/launch-config-forkless-data.j2 + output_file: .docker/launch-config-forkless-data.json + variables: | + FEATURE=${{ matrix.features }} + RUNTIME=${{ matrix.runtime }} + + - name: Show launch-config-forkless configuration + run: cat .docker/launch-config-forkless-data.json + + + - name: Build the stack + run: docker-compose -f ".docker/forking/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" up -d --build --force-recreate --timeout 300 + + - name: Check if docker logs consist logs related to Runtime Upgrade testing. + if: success() + run: | + counter=160 + function do_docker_logs { + docker logs --details parachain-fork 2>&1 + } + function is_started { + echo "Check Docker logs" + DOCKER_LOGS=$(do_docker_logs) + if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then + echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸" + return 0 + exit 0 + fi + echo "Function is_started: Return 1" + return 1 + } + while ! is_started; do + echo "Waiting for special message in log files " + sleep 30s + counter=$(( $counter - 1 )) + echo "Counter: $counter" + if [ "$counter" -gt "0" ]; then + continue + else + break + fi + done + exit 1 + shell: bash + + - name: Collect Docker Logs + if: success() || failure() + uses: jwalton/gh-docker-logs@v2.2.0 + with: + dest: './forkless-parachain-upgrade-data-logs.${{ matrix.features }}.log' + images: 'node-parachain' + + - name: Show Docker logs + run: cat './forkless-parachain-upgrade-data-logs.${{ matrix.features }}.log' + + - name: Stop running containers + if: always() # run this step always + run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down --- a/.github/workflows/forkless-update-nodata.yml +++ b/.github/workflows/forkless-update-nodata.yml @@ -1,4 +1,4 @@ -name: Upgrade nodata +name: upgrade nodata # Controls when the action will run. on: @@ -61,7 +61,7 @@ forkless-update-nodata: needs: prepare-execution-marix # The type of runner that the job will run on - runs-on: [self-hosted-ci,medium] + runs-on: [self-hosted-ci,large] @@ -109,7 +109,7 @@ - name: Generate ENV related extend file for docker-compose uses: cuchi/jinja2-action@v1.2.0 with: - template: .docker/docker-compose.tmp-forkless.j2 + template: .docker/docker-compose.tmp-forkless-nodata.j2 output_file: .docker/docker-compose.${{ matrix.network }}.yml variables: | REPO_URL=${{ github.server_url }}/${{ github.repository }}.git @@ -125,21 +125,21 @@ - name: Show build configuration run: cat .docker/docker-compose.${{ matrix.network }}.yml - - name: Generate launch-config.json + - name: Generate launch-config-forkless-nodata.json uses: cuchi/jinja2-action@v1.2.0 with: - template: .docker/launch-config.j2 - output_file: .docker/launch-config-forkless.json + template: .docker/forkless-config/launch-config-forkless-nodata.j2 + output_file: .docker/launch-config-forkless-nodata.json variables: | FEATURE=${{ matrix.features }} RUNTIME=${{ matrix.runtime }} - name: Show launch-config-forkless configuration - run: cat .docker/launch-config-forkless.json + run: cat .docker/launch-config-forkless-nodata.json - name: Build the stack - run: docker-compose -f ".docker/docker-compose-forkless.yaml" -f ".docker/docker-compose.${{ matrix.network }}.yml" up -d --build --force-recreate --timeout 300 + run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" up -d --build --force-recreate --timeout 300 - name: Check if docker logs consist logs related to Runtime Upgrade testing. if: success() @@ -153,24 +153,25 @@ } function is_started { if [ "$(check_container_status)" == "true" ]; then - echo "Container: node-parachain RUNNING"; - echo "Check Docker logs" - DOCKER_LOGS=$(do_docker_logs) - if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then - echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸" - return 0 - exit 1 - else - echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸 - Not found in logs output." - return 1 - exit 1 - fi + echo "Container: node-parachain RUNNING"; + echo "Check Docker logs" + DOCKER_LOGS=$(do_docker_logs) + if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then + echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸" + return 0 + elif [[ ${DOCKER_LOGS} = *"🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧"* ]];then + echo "🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧" + return 1 + else + echo "Message not found in logs output, repeating..." + return 1 + fi else - echo "Container node-parachain not RUNNING" - echo "Halting all future checks" - exit 1 + echo "Container node-parachain not RUNNING" + echo "Halting all future checks" + exit 1 fi - exit 0 + exit 0 } while ! is_started; do echo "Waiting for special message in log files " @@ -178,33 +179,25 @@ counter=$(( $counter - 1 )) echo "Counter: $counter" if [ "$counter" -gt "0" ]; then - continue + continue else - break + break fi - done - echo "Halting script" - exit 0 + done + echo "Halting script" + exit 0 shell: bash - name: Collect Docker Logs if: success() || failure() uses: jwalton/gh-docker-logs@v2.2.0 with: - dest: './forkless-parachain-update-nodata-logs.${{ matrix.features }}' + dest: './forkless-parachain-upgrade-nodata-logs.${{ matrix.features }}.log' images: 'node-parachain' - - name: Tar logs - if: success() || failure() - run: tar cvzf ./forkless-parachain-update-nodata-logs.${{ matrix.features }}.tgz ./forkless-parachain-update-nodata-logs.${{ matrix.features }} + - name: Show docker logs + run: cat './forkless-parachain-upgrade-nodata-logs.${{ matrix.features }}.log' - - name: Upload logs to GitHub - if: success() || failure() - uses: actions/upload-artifact@master - with: - name: forkless-parachain-update-nodata-logs.${{ matrix.features }}.tgz - path: ./forkless-parachain-update-nodata-logs.${{ matrix.features }}.tgz - - name: Stop running containers if: always() # run this step always - run: docker-compose -f ".docker/docker-compose-forkless.yaml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down + run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down -- gitstuff