difftreelog
ci move forkless-data to baedeker
in: master
2 files changed
.baedeker/forkless-data.jsonnetdiffbeforeafterboth--- /dev/null
+++ b/.baedeker/forkless-data.jsonnet
@@ -0,0 +1,67 @@
+local
+m = import 'baedeker-library/mixin/spec.libsonnet',
+rm = import 'baedeker-library/mixin/raw-spec.libsonnet',
+;
+
+function(relay_spec, forked_spec, fork_source)
+
+local relay = {
+ name: 'relay',
+ bin: 'bin/polkadot',
+ validatorIdAssignment: 'staking',
+ spec: {Genesis:{
+ chain: relay_spec,
+ modify:: m.genericRelay($),
+ }},
+ nodes: {
+ [name]: {
+ bin: $.bin,
+ wantedKeys: 'relay',
+ },
+ for name in ['alice', 'bob', 'charlie', 'dave', 'eve']
+ },
+};
+
+local unique = {
+ name: 'unique',
+ bin: 'bin/unique',
+ paraId: 1001,
+ spec: {Raw:{
+ local modifyRaw = bdk.mixer([
+ rm.resetNetworking($),
+ rm.decodeSpec(),
+ rm.polkaLaunchPara($),
+ rm.reencodeSpec(),
+ ]),
+ raw_spec: modifyRaw({
+ name: "Unused",
+ id: "%s_local" % forked_spec,
+ bootNodes: error "override me",
+ chainType: error "override me",
+ telemetryEndpoints: error "override me",
+ codeSubstitutes: error "override me",
+ para_id: error "override me",
+ relay_chain: "unused",
+ genesis: {
+ raw: {
+ top: cql.chain(fork_source).latest._preloadKeys._raw,
+ childrenDefault: {},
+ },
+ },
+ }),
+ }},
+ nodes: {
+ [name]: {
+ bin: $.bin,
+ wantedKeys: 'para',
+ },
+ for name in ['alice', 'bob']
+ },
+};
+
+relay + {
+ parachains: {
+ [para.name]: para,
+ for para in [unique]
+ },
+}
.github/workflows/forkless-update-data.ymldiffbeforeafterboth1# Forkless update with data replication2# https://cryptousetech.atlassian.net/wiki/spaces/CI/pages/2586869792/Forkless+update+with+data34# Triger: only call from main workflow(re-usable workflows)5on:6 workflow_call:789# A workflow run is made up of one or more jobs that can run sequentially or in parallel10jobs:1112 prepare-execution-marix:1314 name: Prepare execution matrix1516 runs-on: self-hosted-ci17 outputs:18 matrix: ${{ steps.create_matrix.outputs.matrix }}1920 steps:2122 - name: Clean Workspace23 uses: AutoModality/action-clean@v1.1.02425 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it26 - uses: actions/checkout@v3.1.027 with:28 ref: ${{ github.head_ref }} #Checking out head commit2930 - name: Read .env file31 uses: xom9ikk/dotenv@v23233 - name: Create Execution matrix34 uses: CertainLach/create-matrix-action@v435 id: create_matrix36 with:37 # TODO: Remove state-version-0 after mainnet upgrade38 matrix: |39 network {quartz}, wasm_name {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, replica_from_address {${{ env.QUARTZ_REPLICA_FROM }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}, extra_features {,state-version-0}40 network {unique}, wasm_name {unique}, mainnet_branch {${{ env.UNIQUE_MAINNET_BRANCH }}}, replica_from_address {${{ env.UNIQUE_REPLICA_FROM }}}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}, extra_features {,state-version-0}41 network {opal}, wasm_name {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}, replica_from_address {${{ env.OPAL_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}, extra_features {,state-version-0}42 network {sapphire}, wasm_name {quartz}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, replica_from_address {${{ env.SAPPHIRE_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}, extra_features {,state-version-0}4344 forkless-data-build:45 needs: prepare-execution-marix46 # The type of runner that the job will run on47 runs-on: [self-hosted-ci,large]48 timeout-minutes: 13804950 name: ${{ matrix.network }}-data-build51 strategy:52 matrix:53 include: ${{fromJson(needs.prepare-execution-marix.outputs.matrix)}}5455 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.5657 steps:58 - name: Clean Workspace59 uses: AutoModality/action-clean@v1.1.06061 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it62 - uses: actions/checkout@v3.1.063 with:64 ref: ${{ github.head_ref }} #Checking out head commit6566 - name: Read .env file67 uses: xom9ikk/dotenv@v26869 # Prepare SHA 70 - name: Prepare SHA71 uses: ./.github/actions/prepare7273 # Build main image for FORKLESS-UPDATE-NODATA74 - name: Generate ENV related extend Dockerfile file75 uses: cuchi/jinja2-action@v1.2.076 with:77 template: .docker/Dockerfile-parachain-upgrade-data.j278 output_file: .docker/Dockerfile-parachain-upgrade-data.${{ matrix.network }}.yml79 variables: |80 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}81 NETWORK=${{ matrix.network }}82 MAINNET_BRANCH=${{ matrix.mainnet_branch }}83 WASM_NAME=${{ matrix.wasm_name }}84 RELAY_CHAIN_TYPE=${{ env.RELAY_CHAIN_TYPE }}85 DESTINATION_SPEC_VERSION=${{ env.DESTINATION_SPEC_VERSION }}86 POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}87 REPLICA_FROM=${{ matrix.replica_from_address }}88 CHAINQL=${{ env.CHAINQL }}89 EXTRA_FEATURES=${{ matrix.extra_features }}9091 - name: Show build configuration92 run: cat .docker/Dockerfile-parachain-upgrade-data.${{ matrix.network }}.yml9394 - name: Run find-and-replace to remove slashes from branch name95 uses: mad9000/actions-find-and-replace-string@496 id: branchname97 with:98 source: ${{ github.head_ref }}99 find: '/'100 replace: '-'101102 - name: Set build SHA103 shell: bash104 run: |105 echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV106107 - name: Build the stack108 run: cd .docker/ && docker build --no-cache --file ./Dockerfile-parachain-upgrade-data.${{ matrix.network }}.yml --tag uniquenetwork/ci-forkless-data-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA ../109110 - name: Log in to Docker Hub111 uses: docker/login-action@v2.1.0112 with:113 username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}114 password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}115116 - name: Push docker image version117 run: docker push uniquenetwork/ci-forkless-data-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA118119 - name: Remove builder cache120 if: always() # run this step always121 run: |122 docker builder prune -f123 docker system prune -f124125 forkless-data-tests:126 needs: [prepare-execution-marix, forkless-data-build]127 # The type of runner that the job will run on128 runs-on: [self-hosted-ci, large]129130 timeout-minutes: 600131132 name: ${{ matrix.network }}-data-tests133134 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.135136 strategy:137 matrix:138 include: ${{fromJson(needs.prepare-execution-marix.outputs.matrix)}}139140 steps:141 - name: Skip if pull request is in Draft142 if: github.event.pull_request.draft == true143 run: exit 1144145 - name: Clean Workspace146 uses: AutoModality/action-clean@v1.1.0147148 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it149 - uses: actions/checkout@v3.1.0150 with:151 ref: ${{ github.head_ref }} #Checking out head commit152153 - name: Prepare154 uses: ./.github/actions/prepare155156 - name: Set build SHA157 shell: bash158 run: |159 echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV160161 - name: Run find-and-replace to remove slashes from branch name162 uses: mad9000/actions-find-and-replace-string@4163 id: branchname164 with:165 source: ${{ github.head_ref }}166 find: '/'167 replace: '-'168169 - name: Read .env file170 uses: xom9ikk/dotenv@v2171172 - name: Generate ENV related extend file for docker-compose173 uses: cuchi/jinja2-action@v1.2.0174 with:175 template: .docker/docker-compose.forkless-data.j2176 output_file: .docker/docker-compose.forkless-data.${{ matrix.network }}.yml177 variables: |178 NETWORK=${{ matrix.network }}179 BUILD_TAG=${{ steps.branchname.outputs.value }}-$BUILD_SHA180181 - name: Show build configuration182 run: cat .docker/docker-compose.forkless-data.${{ matrix.network }}.yml183184 - name: Log in to Docker Hub185 uses: docker/login-action@v2.1.0186 with:187 username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}188 password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}189190 - name: Build the stack191 run: docker-compose -f ".docker/docker-compose.forkless-data.${{ matrix.network }}.yml" up -d --remove-orphans --force-recreate --timeout 300192193 - uses: actions/setup-node@v3.5.1194 with:195 node-version: 18196197 - name: Check if docker logs consist logs related to Runtime Upgrade testing.198 if: success()199 run: |200 counter=160201 function check_container_status {202 docker inspect -f {{.State.Running}} forkless-data-${{ matrix.network }}203 }204 function do_docker_logs {205 docker logs --details forkless-data-${{ matrix.network }} 2>&1206 }207 function is_started {208 if [ "$(check_container_status)" == "true" ]; then209 echo "Container: forkless-data-${{ matrix.network }} RUNNING";210 echo "Check Docker logs"211 DOCKER_LOGS=$(do_docker_logs)212 if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then213 echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"214 return 0215 elif [[ ${DOCKER_LOGS} = *"🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧"* ]];then216 echo "🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧"217 return 1218 else219 echo "Message not found in logs output, repeating..."220 return 1221 fi222 else223 echo "Container forkless-data-${{ matrix.network }} not RUNNING"224 echo "Halting all future checks"225 exit 1226 fi227 exit 0228 }229 while ! is_started; do230 echo "Waiting for special message in log files "231 sleep 30s232 counter=$(( $counter - 1 ))233 echo "Counter: $counter"234 if [ "$counter" -gt "0" ]; then235 continue236 else237 echo "Counter reached zero, yet upgrade is not finished"238 exit 1239 fi240 done241 echo "Halting script"242 exit 0243 shell: bash244245 - name: Collect Docker Logs246 if: success() || failure()247 uses: jwalton/gh-docker-logs@v2.2.1248 with:249 dest: './forkless-parachain-upgrade-data-logs.${{ matrix.network }}'250251 - name: Show Docker logs252 if: success() || failure()253 run: cat './forkless-parachain-upgrade-data-logs.${{ matrix.network }}/forkless-data-${{ matrix.network }}.log'254255 - name: Stop running containers256 if: always() # run this step always257 run: docker-compose -f ".docker/docker-compose.forkless-data.${{ matrix.network }}.yml" down --volumes258259 - name: Remove builder cache260 if: always() # run this step always261 run: |262 docker builder prune -f -a263 docker system prune -f264 docker image prune -f -a1# Forkless update with data replication2# https://cryptousetech.atlassian.net/wiki/spaces/CI/pages/2586869792/Forkless+update+with+data34# Triger: only call from main workflow(re-usable workflows)5on:6 workflow_call:789# A workflow run is made up of one or more jobs that can run sequentially or in parallel10jobs:1112 prepare-execution-matrix:1314 name: execution matrix1516 runs-on: self-hosted-ci17 outputs:18 matrix: ${{ steps.create_matrix.outputs.matrix }}1920 steps:2122 - name: Clean Workspace23 uses: AutoModality/action-clean@v1.1.02425 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it26 - uses: actions/checkout@v3.1.027 with:28 ref: ${{ github.head_ref }} #Checking out head commit2930 - name: Read .env file31 uses: xom9ikk/dotenv@v23233 - name: Create Execution matrix34 uses: CertainLach/create-matrix-action@v435 id: create_matrix36 with:37 matrix: |38 network {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}, runtime_features {opal-runtime}, fork_source {${{ env.OPAL_REPLICA_FROM }}}39 network {sapphire}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}, runtime_features {sapphire-runtime}, wasm_name {quartz}, fork_source {${{ env.SAPPHIRE_REPLICA_FROM }}}40 network {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}, runtime_features {quartz-runtime}, wasm_name {quartz}, fork_source {${{ env.QUARTZ_REPLICA_FROM }}}41 network {unique}, mainnet_branch {${{ env.UNIQUE_MAINNET_BRANCH }}}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}, runtime_features {unique-runtime}, wasm_name {unique}, fork_source {${{ env.UNIQUE_REPLICA_FROM }}}4243 forkless-data-build:4445 needs: prepare-execution-matrix46 # The type of runner that the job will run on47 runs-on: [self-hosted-ci,large]4849 timeout-minutes: 13805051 name: ${{ matrix.network }}-data-build5253 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.5455 strategy:56 matrix:57 include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}}5859 steps:6061 - name: Clean Workspace62 uses: AutoModality/action-clean@v1.1.06364 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it65 - uses: actions/checkout@v3.1.066 with:67 ref: ${{ github.head_ref }} #Checking out head commit6869 # Prepare SHA 70 - name: Prepare SHA71 uses: ./.github/actions/prepare7273 - name: Read .env file74 uses: xom9ikk/dotenv@v27576 - name: Log in to Docker Hub77 uses: docker/login-action@v2.1.078 with:79 username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}80 password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}8182 - name: Generate ENV related extend Dockerfile file for POLKADOT83 uses: cuchi/jinja2-action@v1.2.084 with:85 template: .docker/Dockerfile-polkadot.j286 output_file: .docker/Dockerfile-polkadot.${{ matrix.relay_branch }}.yml87 variables: |88 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}89 POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}90 91 - name: Prepare polkadot92 uses: ./.github/actions/buildContainer93 id: polkadot94 with:95 container: uniquenetwork/builder-polkadot96 tag: ${{ matrix.relay_branch }}97 context: .docker98 dockerfile: Dockerfile-polkadot.${{ matrix.relay_branch }}.yml99 dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}100 dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }}101102 - name: Prepare mainnet103 uses: ./.github/actions/buildContainer104 id: mainnet105 with:106 container: uniquenetwork/ci-data-${{ matrix.network }}107 tag: ${{ matrix.mainnet_branch }}108 context: .docker109 dockerfile: Dockerfile-unique-release110 args: |111 --build-arg RUNTIME_FEATURES=${{ matrix.runtime_features }}112 --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}113 --build-arg UNIQUE_VERSION=${{ matrix.mainnet_branch }}114 dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}115 dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }}116117 - name: Prepare latest118 uses: ./.github/actions/buildContainer119 id: latest120 with:121 container: uniquenetwork/ci-data-${{ matrix.network }}122 tag: ${{ env.REF_SLUG }}-${{ env.BUILD_SHA }}123 context: .124 dockerfile: .docker/Dockerfile-unique125 args: |126 --build-arg RUNTIME_FEATURES=${{ matrix.runtime_features }}127 --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}128 dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}129 dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }}130131 - name: Extract wasms132 uses: ./.github/actions/extractDocker133 id: wasms134 with:135 image: ${{ steps.latest.outputs.name }}136 directory: /wasm137138 - uses: actions/setup-node@v3.5.1139 with:140 node-version: 18141142 - name: Install baedeker143 uses: UniqueNetwork/baedeker-action/setup@built144145 - name: Setup library146 run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library147148 - name: Start network149 uses: UniqueNetwork/baedeker-action@built150 id: bdk151 with:152 jpath: |153 .baedeker/vendor154 tla-str: |155 relay_spec=rococo-local156 forked_spec=${{ matrix.network }}157 fork_source=${{ matrix.fork_source }}158 inputs: |159 .baedeker/forkless-data.jsonnet160 snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.name }}'}})161 snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}, extra_node_mixin = {legacyRpc: true})162163 - name: "Reconcile: runtime is upgraded"164 working-directory: tests165 run: |166 yarn167 ./scripts/wait_for_first_block.sh168 echo "Executing upgrade"169 yarn ts-node --esm src/util/authorizeEnactUpgrade.ts ${{ steps.wasms.outputs.dir }}/${{ matrix.wasm_name }}-runtime/${{ matrix.wasm_name }}_runtime.compact.compressed.wasm170 env:171 RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}172173 - name: Run Parallel tests after forkless upgrade174 working-directory: tests175 run: |176 ./scripts/wait_for_first_block.sh177 echo "Ready to start tests"178 NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW}179 env:180 RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}181182 - name: Run Sequential tests after forkless upgrade183 if: success() || failure()184 working-directory: ${{ matrix.mainnet_branch }}/tests185 run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW}186 env:187 RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}188189 - name: Remove builder cache190 if: always() # run this step always191 run: |192 docker system prune -f