git.delta.rocks / unique-network / refs/commits / f0f9c65ec96f

difftreelog

Merge pull request #489 from UniqueNetwork/CI-42-fork-update-parachain-relay-withdata

Yaroslav Bolyukin2022-08-15parents: #1ad8b2e #d68e1f7.patch.diff
in: master
add CI for update parachain & relay with forking data

8 files changed

added.docker/forking/Dockerfile-parachain-live-forkdiffbeforeafterboth
--- /dev/null
+++ b/.docker/forking/Dockerfile-parachain-live-fork
@@ -0,0 +1,102 @@
+# ===== Rust builder =====
+FROM ubuntu:20.04 as rust-builder
+LABEL maintainer="Unique.Network"
+
+ARG RUST_TOOLCHAIN=
+
+ENV RUST_TOOLCHAIN $RUST_TOOLCHAIN
+
+ENV CARGO_HOME="/cargo-home"
+ENV PATH="/cargo-home/bin:$PATH"
+ENV TZ=UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt-get update && \
+    apt-get install -y curl cmake pkg-config libssl-dev git clang && \
+    apt-get clean && \
+    rm -r /var/lib/apt/lists/*
+
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain none
+
+RUN rustup toolchain uninstall $(rustup toolchain list) && \
+    rustup toolchain install $RUST_TOOLCHAIN && \
+    rustup default $RUST_TOOLCHAIN && \
+    rustup target list --installed && \
+    rustup show
+RUN rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN
+
+# ===== BUILD target version ======
+FROM rust-builder as builder-unique-target
+
+ARG PROFILE=release
+ARG FEATURE=
+ARG BRANCH=
+ARG REPO_URL=
+
+RUN mkdir /unique_parachain
+WORKDIR /unique_parachain
+
+RUN git clone $REPO_URL -b $BRANCH . && \
+    cargo build --features=$FEATURE --$PROFILE
+
+# ===== BUILD POLKADOT =====
+FROM rust-builder as builder-polkadot
+
+ARG POLKADOT_BUILD_BRANCH=
+ENV POLKADOT_BUILD_BRANCH $POLKADOT_BUILD_BRANCH
+
+RUN mkdir /unique_parachain
+WORKDIR /unique_parachain
+
+RUN git clone -b $POLKADOT_BUILD_BRANCH --depth 1 https://github.com/paritytech/polkadot.git && \
+    cd polkadot && \
+    cargo build --release
+
+# ===== BUILD CHAINQL =====
+FROM rust-builder as builder-chainql
+
+RUN mkdir chainql
+WORKDIR /chainql
+
+RUN git clone --depth 1 https://github.com/CertainLach/chainql.git . && \
+    cargo build --release
+
+# ===== RUN ======
+
+FROM ubuntu:20.04
+
+ARG RUNTIME=
+ENV RUNTIME $RUNTIME
+
+RUN apt-get -y update && \
+      apt-get -y install curl git && \
+      curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash && \
+      export NVM_DIR="$HOME/.nvm" && \
+      [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \
+      nvm install v16.16.0 && \
+      nvm use v16.16.0
+
+RUN git clone https://github.com/uniquenetwork/polkadot-launch -b feature/parachain-forking
+
+RUN export NVM_DIR="$HOME/.nvm" && \
+    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \
+    cd /polkadot-launch && \
+    npm install --global yarn && \
+    yarn install
+
+RUN echo "$RUNTIME"
+
+COPY --from=builder-unique-target /unique_parachain/target/release/unique-collator /unique-chain/target/release/
+
+COPY --from=builder-polkadot /unique_parachain/polkadot/target/release/polkadot /polkadot/target/release/
+
+COPY --from=builder-chainql /chainql/target/release/chainql /chainql/target/release/
+
+ARG FORK_FROM=
+ENV FORK_FROM=$FORK_FROM
+CMD export NVM_DIR="$HOME/.nvm" PATH="$PATH:/chainql/target/release" FORK_FROM && \
+    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \
+    cd /polkadot-launch && \
+    yarn start launch-config.json
+
+
added.docker/forking/docker-compose-fork.yamldiffbeforeafterboth
--- /dev/null
+++ b/.docker/forking/docker-compose-fork.yaml
@@ -0,0 +1,28 @@
+version: "3.5"
+
+services:
+  parachain-fork:
+    build:
+      context: ../
+      dockerfile: .docker/forking/Dockerfile-parachain-live-fork
+    image: parachain-fork
+    container_name: parachain-fork
+    volumes:
+       - type: bind
+         source: ./launch-config-fork.json
+         target: /polkadot-launch/launch-config.json
+         read_only: true
+       - type: bind
+         source: ./fork.jsonnet
+         target: /polkadot-launch/fork.jsonnet
+         read_only: true          
+    expose:
+      - 9944
+      - 9933
+    ports:
+      - 127.0.0.1:9944:9944
+      - 127.0.0.1:9933:9933
+    logging:
+      options:
+        max-size: "1m"
+        max-file: "3"
added.docker/forking/docker-compose.tmp-fork.j2diffbeforeafterboth
--- /dev/null
+++ b/.docker/forking/docker-compose.tmp-fork.j2
@@ -0,0 +1,15 @@
+version: "3.5"
+
+services:
+  parachain-fork:
+    build:
+      args:
+        - "RUST_TOOLCHAIN={{ RUST_TOOLCHAIN }}"
+        - "BRANCH={{ BRANCH }}"
+        - "REPO_URL={{ REPO_URL }}"
+        - "FEATURE={{ FEATURE }}"
+        - "RUNTIME={{ RUNTIME }}"
+        - "POLKADOT_BUILD_BRANCH={{ POLKADOT_BUILD_BRANCH }}"
+        - "MAINNET_TAG={{ MAINNET_TAG }}"
+        - "MAINNET_BRANCH={{ MAINNET_BRANCH }}"
+        - "FORK_FROM={{ FORK_FROM }}"
added.docker/forking/fork.jsonnetdiffbeforeafterboth
--- /dev/null
+++ b/.docker/forking/fork.jsonnet
@@ -0,0 +1,59 @@
+
+function(rawSpec, forkFrom)
+local sourceChain = cql.chain(forkFrom).latest;
+
+local raw = local sourceRaw = sourceChain._raw._preloadKeys; {
+  [key]: cql.toHex(sourceRaw[key])
+  for key in std.objectFields(sourceRaw)
+  if sourceRaw[key] != null
+};
+
+local
+auraKeys = [
+	// AuraExt.Authorities, we don't have aura pallet enabled for some reason, to refer using cql api
+	'0x3c311d57d4daf52904616cf69648081e5e0621c4869aa60c02be9adcc98a0d1d',
+	// Aura.Authorities
+	'0x57f8dc2f5ab09467896f47300f0424385e0621c4869aa60c02be9adcc98a0d1d',
+],
+
+// Keys, which should be migrated from passed spec, rather than from forked chain
+wantedKeys = [
+	sourceChain.ParachainInfo._key.ParachainId,
+	sourceChain.Sudo._key.Key,
+	sourceChain.System.BlockHash._key['0'],
+	sourceChain.System._key.ParentHash,
+] + auraKeys,
+
+// Keys to remove from original chain
+unwantedPrefixes = [
+	// Aura.CurrentSlot
+	'0x57f8dc2f5ab09467896f47300f04243806155b3cd9a8c9e5e9a23fd5dc13a5ed',
+	// Ensure there will be no panics caused by unexpected kept state
+	sourceChain.ParachainSystem._key.ValidationData,
+	sourceChain.ParachainSystem._key.RelayStateProof,
+	sourceChain.ParachainSystem._key.HostConfiguration,
+	sourceChain.ParachainSystem._key.LastDmqMqcHead,
+	// Part of head
+	sourceChain.System._key.BlockHash,
+	sourceChain.System._key.Number,
+	sourceChain.System._key.Digest,
+] + auraKeys,
+
+cleanupRaw(raw) = {
+	[key]: raw[key]
+	for key in std.objectFields(raw)
+	if std.all(std.map(function(prefix) !std.startsWith(key, prefix), unwantedPrefixes))
+};
+
+local originalRaw = rawSpec.genesis.raw.top;
+local outSpec = rawSpec {
+	genesis+: {
+		raw+: {
+			top: cleanupRaw(raw) {
+				[key]: originalRaw[key]
+				for key in wantedKeys
+			},
+		},
+	},
+};
+outSpec
added.docker/forking/launch-config-fork.j2diffbeforeafterboth
--- /dev/null
+++ b/.docker/forking/launch-config-fork.j2
@@ -0,0 +1,128 @@
+{
+      "relaychain": {
+        "bin": "/polkadot/target/release/polkadot",
+        "chain": "westend-local",
+        "nodes": [
+            {
+                "name": "alice",
+                "wsPort": 9844,
+                "rpcPort": 9843,
+                "port": 30444,
+                "flags": [
+                    "-lparachain::candidate_validation=debug",
+                    "-lxcm=trace",
+                    "--rpc-cors=all",
+                    "--unsafe-rpc-external",
+                    "--unsafe-ws-external"
+                ]
+            },
+            {
+                "name": "bob",
+                "wsPort": 9855,
+                "rpcPort": 9854,
+                "port": 30555,
+                "flags": [
+                    "-lparachain::candidate_validation=debug",
+                    "-lxcm=trace",
+                    "--rpc-cors=all",
+                    "--unsafe-rpc-external",
+                    "--unsafe-ws-external"
+                ]
+            },
+            {
+                "name": "charlie",
+                "wsPort": 9866,
+                "rpcPort": 9865,
+                "port": 30666,
+                "flags": [
+                    "-lparachain::candidate_validation=debug",
+                    "-lxcm=trace",
+                    "--rpc-cors=all",
+                    "--unsafe-rpc-external",
+                    "--unsafe-ws-external"
+                ]
+            },
+            {
+                "name": "dave",
+                "wsPort": 9877,
+                "rpcPort": 9876,
+                "port": 30777,
+                "flags": [
+                    "-lparachain::candidate_validation=debug",
+                    "-lxcm=trace",
+                    "--rpc-cors=all",
+                    "--unsafe-rpc-external",
+                    "--unsafe-ws-external"
+                ]
+            },
+            {
+                "name": "eve",
+                "wsPort": 9888,
+                "rpcPort": 9887,
+                "port": 30888,
+                "flags": [
+                    "-lparachain::candidate_validation=debug",
+                    "-lxcm=trace",
+                    "--rpc-cors=all",
+                    "--unsafe-rpc-external",
+                    "--unsafe-ws-external"
+                ]
+            }
+        ],
+        "genesis": {
+            "runtime": {
+                "runtime_genesis_config": {
+                    "parachainsConfiguration": {
+                        "config": {
+                            "validation_upgrade_frequency": 1,
+                            "validation_upgrade_delay": 1
+                        }
+                    }
+                }
+            }
+        }
+    },
+      "parachains": [
+        {
+            "bin": "/unique-chain/current/release/unique-collator",
+            "id": "1000",
+            "balance": "1000000000000000000000000",
+            "chainRawInitializer": [
+                "chainql",
+                "--ext-str=FORK_FROM",
+                "--tla-code=rawSpec=import '${rawSpec}'",
+                "--tla-code=forkFrom=std.extVar('FORK_FROM')",
+                "fork.jsonnet"
+            ],
+            "nodes": [
+                {
+                    "port": 31200,
+                    "wsPort": 9944,
+                    "rpcPort": 9933,
+                    "name": "alice",
+                    "flags": [
+                        "--rpc-cors=all",
+                        "--unsafe-rpc-external",
+                        "--unsafe-ws-external",
+                        "-lxcm=trace"
+                    ]
+                },
+                {
+                    "port": 31201,
+                    "wsPort": 9945,
+                    "rpcPort": 9934,
+                    "name": "bob",
+                    "flags": [
+                        "--rpc-cors=all",
+                        "--unsafe-rpc-external",
+                        "--unsafe-ws-external",
+                        "-lxcm=trace"
+                    ]
+                }
+            ]
+        }
+    ],
+    "simpleParachains": [],
+    "hrmpChannels": [],
+    "finalization": false
+}
modified.dockerignorediffbeforeafterboth
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,6 +1,5 @@
 .git/
 .github/
-.docker/
 doc/
 target/
 tests/
modified.envdiffbeforeafterboth
--- a/.env
+++ b/.env
@@ -5,4 +5,4 @@
 UNIQUE_MAINNET_TAG=v924010
 
 KUSAMA_MAINNET_BRANCH=release-v0.9.26
-QUARTZ_MAINNET_TAG=v924012
\ No newline at end of file
+QUARTZ_MAINNET_TAG=quartz-v924012-2
added.github/workflows/fork-update-withdata.ymldiffbeforeafterboth
after · .github/workflows/fork-update-withdata.yml
1name: Fork Parachain update with data23# Controls when the action will run.4on:5 # Triggers the workflow on push or pull request events but only for the master branch6  pull_request:7    branches:8      - master9    types:10      - opened11      - reopened12      - synchronize   #commit(s) pushed to the pull request1314  # Allows you to run this workflow manually from the Actions tab15  workflow_dispatch:1617#Define Workflow variables18env:19  REPO_URL: ${{ github.server_url }}/${{ github.repository }}2021# A workflow run is made up of one or more jobs that can run sequentially or in parallel22jobs:2324  prepare-execution-marix-fork:25    26    name: Prepare execution matrix for fork27    28    runs-on: self-hosted-ci29    outputs:30      matrix: ${{ steps.create_matrix.outputs.matrix }}31    steps:32      33      - name: Clean Workspace34        uses: AutoModality/action-clean@v1.1.03536      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it37      - uses: actions/checkout@v338        with:39          ref: ${{ github.head_ref }}  #Checking out head commit4041      - name: Read .env file42        uses: xom9ikk/dotenv@v1.0.24344      - name: Create Execution matrix45        uses: fabiocaccamo/create-matrix-action@v246        id: create_matrix47        with:48          matrix: |49            network {Opal}, runtime {opal}, features {opal-runtime}, mainnet_branch {${{ env.OPAL_MAINNET_TAG }}}, fork_from_address {wss://eu-ws-opal.unique.network:443}50            network {Quartz}, runtime {quartz}, features {quartz-runtime}, mainnet_branch {${{ env.QUARTZ_MAINNET_TAG }}}, fork_from_address {wss://eu-ws-quartz.unique.network:443}51            network {Unique}, runtime {unique}, features {unique-runtime}, mainnet_branch {${{ env.UNIQUE_MAINNET_TAG }}}, fork_from_address {wss://eu-ws.unique.network:443}52535455  fork-update-withdata:56    needs: prepare-execution-marix-fork57    # The type of runner that the job will run on58    runs-on: self-hosted-ci5960    name: Build Container, Spin it Up an test6162    continue-on-error: true         #Do not stop testing of matrix runs failed.  As it decided during PR review - it required 50/50& Let's check it with false.6364    strategy:65      matrix:66        include: ${{fromJson(needs.prepare-execution-marix-fork.outputs.matrix)}}6768    steps:69      - name: Skip if pull request is in Draft70        # `if: github.event.pull_request.draft == true` should be kept here, at71        # the step level, rather than at the job level. The latter is not72        # recommended because when the PR is moved from "Draft" to "Ready to73        # review" the workflow will immediately be passing (since it was skipped),74        # even though it hasn't actually ran, since it takes a few seconds for75        # the workflow to start. This is also disclosed in:76        # https://github.community/t/dont-run-actions-on-draft-pull-requests/16817/1777        # That scenario would open an opportunity for the check to be bypassed:78        # 1. Get your PR approved79        # 2. Move it to Draft80        # 3. Push whatever commits you want81        # 4. Move it to "Ready for review"; now the workflow is passing (it was82        #    skipped) and "Check reviews" is also passing (it won't be updated83        #    until the workflow is finished)84        if: github.event.pull_request.draft == true85        run: exit 18687      - name: Clean Workspace88        uses: AutoModality/action-clean@v1.1.08990      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it91      - uses: actions/checkout@v392        with:93          ref: ${{ github.head_ref }}  #Checking out head commit9495      - name: Read .env file96        uses: xom9ikk/dotenv@v1.0.29798      - name: Generate ENV related extend file for docker-compose99        uses: cuchi/jinja2-action@v1.2.0100        with:101          template: .docker/forking/docker-compose.tmp-fork.j2102          output_file: .docker/forking/docker-compose.${{ matrix.network }}.yml103          variables: |104            REPO_URL=${{ github.server_url }}/${{ github.repository }}.git105            RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}106            POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }}107            POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }}108            MAINNET_TAG=${{ matrix.mainnet_tag }}109            MAINNET_BRANCH=${{ matrix.mainnet_branch }}110            FEATURE=${{ matrix.features }}111            RUNTIME=${{ matrix.runtime }}112            BRANCH=${{ github.head_ref }}113            FORK_FROM=${{ matrix.fork_from_address }}114115      - name: Show build configuration116        run: cat .docker/forking/docker-compose.${{ matrix.network }}.yml117118      - name: Generate launch-config-fork.json119        uses: cuchi/jinja2-action@v1.2.0120        with:121          template: .docker/forking/launch-config-fork.j2122          output_file: .docker/forking/launch-config-fork.json123          variables: |124            FEATURE=${{ matrix.features }}125            RUNTIME=${{ matrix.runtime }}126127      - name: Show launch-config-fork configuration128        run: cat .docker/forking/launch-config-fork.json129130131      - name: Build the stack132        run: docker-compose -f ".docker/forking/docker-compose-fork.yaml" -f ".docker/forking/docker-compose.${{ matrix.network }}.yml" up -d --build  --force-recreate --timeout 300133134      - name: Check if docker logs consist logs related to Runtime Upgrade testing.135        if: success()136        run: |137            counter=160138            function do_docker_logs {139                docker logs --details parachain-fork  2>&1140            }141            function is_started {142                echo "Check Docker logs"143                DOCKER_LOGS=$(do_docker_logs)144                if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then145                     echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"146                     return 0147                     exit 0148                fi149                     echo "Function is_started: Return 1"150                     return 1151            }152            while ! is_started; do153                echo "Waiting for special message in log files "154                sleep 30s155                counter=$(( $counter - 1 ))156                echo "Counter: $counter"157                if [ "$counter" -gt "0" ]; then158                     continue159                else160                     break161                fi162            done163            exit 1164        shell: bash165166      - name: Collect Docker Logs167        if: success() || failure()168        uses: jwalton/gh-docker-logs@v2.2.0169        with:170          dest: './fork-parachain-update-withdata-logs.${{ matrix.features }}'171          images: 'parachain-fork' 172173      - name: Tar logs174        if: success() || failure()175        run: tar cvzf ./fork-parachain-update-withdata-logs.${{ matrix.features }}.tgz ./fork-parachain-update-withdata-logs.${{ matrix.features }}176177      - name: Upload logs to GitHub178        if: success() || failure()179        uses: actions/upload-artifact@master180        with:181          name: fork-parachain-update-withdata-logs.${{ matrix.features }}.tgz182          path: ./fork-parachain-update-withdata-logs.${{ matrix.features }}.tgz183184      - name: Stop running containers185        if: always()                   # run this step always186        run: docker-compose -f ".docker/forking/docker-compose-fork.yaml" -f ".docker/forking/docker-compose.${{ matrix.network }}.yml" down