difftreelog
ci move collator selection to baedeker (#979)
in: master
* modify collator selection workflow for baedeker * fix collator selection workflow for baedeker * add modify in spec for collator selection workflow * test: also accept sapphire as substrateUrl * modify workflow collator-selection for baedeker * ci: remove unused collator-selection setups ---------
4 files changed
.baedeker/collator-selection.jsonnetdiffbeforeafterboth1local2m = import 'baedeker-library/mixin/spec.libsonnet',3;45local relay = {6 name: 'relay',7 bin: 'bin/polkadot',8 validatorIdAssignment: 'staking',9 spec: {Genesis:{10 chain: 'rococo-local',11 modify:: m.genericRelay($),12 }},13 nodes: {14 [name]: {15 bin: $.bin,16 wantedKeys: 'relay',17 },18 for name in ['alice', 'bob', 'charlie', 'dave', 'eve']19 },20};2122local unique = {23 name: 'unique',24 bin: 'bin/unique',25 paraId: 1001,26 spec: {Genesis:{27 modify:: m.genericPara($),28 }},29 nodes: {30 [name]: {31 bin: $.bin,32 wantedKeys: 'para',33 },34 for name in ['alice', 'bob']35 },36};3738relay + {39 parachains: {40 [para.name]: para,41 for para in [unique]42 },43}.github/workflows/collator-selection.ymldiffbeforeafterboth--- a/.github/workflows/collator-selection.yml
+++ b/.github/workflows/collator-selection.yml
@@ -1,15 +1,24 @@
name: collator-selection-test-run
-# Triger: only call from main workflow(re-usable workflows)
+# Controls when the action will run.
on:
workflow_call:
+ # Allows you to run this workflow manually from the Actions tab
+ workflow_dispatch:
+
+#Define Workflow variables
+env:
+ REPO_URL: ${{ github.server_url }}/${{ github.repository }}
+
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
+
prepare-execution-marix:
+
name: Prepare execution matrix
- runs-on: self-hosted-ci
+ runs-on: [self-hosted-ci]
outputs:
matrix: ${{ steps.create_matrix.outputs.matrix }}
@@ -19,7 +28,7 @@
uses: AutoModality/action-clean@v1.1.0
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v3.1.0
with:
ref: ${{ github.head_ref }} #Checking out head commit
@@ -27,99 +36,24 @@
uses: xom9ikk/dotenv@v2
- name: Create Execution matrix
- uses: CertainLach/create-matrix-action@v3
+ uses: CertainLach/create-matrix-action@v4
id: create_matrix
with:
matrix: |
- network {opal}, wasm_name {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}, replica_from_address {${{ env.OPAL_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}
- network {quartz}, wasm_name {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, replica_from_address {${{ env.QUARTZ_REPLICA_FROM }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}
- network {sapphire}, wasm_name {quartz}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, replica_from_address {${{ env.SAPPHIRE_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}
+ network {opal}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}
+ network {quartz}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}
+ network {sapphire}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}
+
+ collator-selection:
- collator-selection-build:
needs: prepare-execution-marix
# The type of runner that the job will run on
runs-on: [self-hosted-ci,large]
- timeout-minutes: 1380
-
- name: ${{ matrix.network }}-build
- strategy:
- matrix:
- include: ${{fromJson(needs.prepare-execution-marix.outputs.matrix)}}
-
- continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.
-
- steps:
- - name: Clean Workspace
- uses: AutoModality/action-clean@v1.1.0
-
- # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- - uses: actions/checkout@v3
- with:
- ref: ${{ github.head_ref }} #Checking out head commit
-
- - name: Read .env file
- uses: xom9ikk/dotenv@v2
-
- # Prepare SHA
- - name: Prepare SHA
- uses: ./.github/actions/prepare
-
- # Build main image for FORKLESS-UPDATE-NODATA
- - name: Generate ENV related extend Dockerfile file
- uses: cuchi/jinja2-action@v1.2.0
- with:
- template: .docker/Dockerfile-collators.j2
- output_file: .docker/Dockerfile-collators.${{ matrix.network }}.yml
- variables: |
- RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}
- NETWORK=${{ matrix.network }}
- POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }}
- POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}
- BRANCH=${{ github.head_ref }}
- - name: Show build configuration
- run: cat .docker/Dockerfile-collators.${{ matrix.network }}.yml
-
- - name: Run find-and-replace to remove slashes from branch name
- uses: mad9000/actions-find-and-replace-string@4
- id: branchname
- with:
- source: ${{ github.head_ref }}
- find: '/'
- replace: '-'
-
- - name: Set build SHA
- shell: bash
- run: |
- echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV
+ timeout-minutes: 1380
- - name: Build the stack
- run: cd .docker/ && docker build --no-cache --file ./Dockerfile-collators.${{ matrix.network }}.yml --tag uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA ../
-
- - name: Log in to Docker Hub
- uses: docker/login-action@v2.1.0
- with:
- username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}
- password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}
-
- - name: Push docker image version
- run: docker push uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA
-
- - name: Remove builder cache
- if: always() # run this step always
- run: |
- docker builder prune -f
- docker system prune -f
-
- collator-selection-tests:
- needs: [prepare-execution-marix, collator-selection-build]
- # The type of runner that the job will run on
- runs-on: [self-hosted-ci, large]
-
- timeout-minutes: 600
+ name: ${{ matrix.network }}
- name: ${{ matrix.network }}-tests
-
continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.
strategy:
@@ -139,13 +73,18 @@
with:
ref: ${{ github.head_ref }} #Checking out head commit
- - name: Prepare
+ # Prepare SHA
+ - name: Prepare SHA
uses: ./.github/actions/prepare
- - name: Set build SHA
- shell: bash
- run: |
- echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV
+ - name: Read .env file
+ uses: xom9ikk/dotenv@v2
+
+ - name: Log in to Docker Hub
+ uses: docker/login-action@v2.1.0
+ with:
+ username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}
+ password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}
- name: Run find-and-replace to remove slashes from branch name
uses: mad9000/actions-find-and-replace-string@4
@@ -155,109 +94,114 @@
find: '/'
replace: '-'
- - name: Read .env file
- uses: xom9ikk/dotenv@v2
-
- - name: Generate ENV related extend file for docker-compose
+ - name: Generate ENV related extend Dockerfile file for POLKADOT
uses: cuchi/jinja2-action@v1.2.0
with:
- template: .docker/docker-compose.collators.j2
- output_file: .docker/docker-compose.collators.${{ matrix.network }}.yml
+ template: .docker/Dockerfile-polkadot.j2
+ output_file: .docker/Dockerfile-polkadot.${{ matrix.relay_branch }}.yml
variables: |
- NETWORK=${{ matrix.network }}
- BUILD_TAG=${{ steps.branchname.outputs.value }}-$BUILD_SHA
+ RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}
+ POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}
+
+ - name: Prepare polkadot
+ uses: ./.github/actions/buildContainer
+ id: polkadot
+ with:
+ container: uniquenetwork/builder-polkadot
+ tag: ${{ matrix.relay_branch }}
+ context: .docker
+ dockerfile: Dockerfile-polkadot.${{ matrix.relay_branch }}.yml
+ dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }}
- - name: Show build configuration
- run: cat .docker/docker-compose.collators.${{ matrix.network }}.yml
+ # - name: pull needed images
+ # run: |
+ # docker pull uniquenetwork/builder-polkadot:${{ matrix.relay_branch }}
- - name: Log in to Docker Hub
- uses: docker/login-action@v2.1.0
- with:
- username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}
- password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}
+ # - name: Build unique-chain
+ # run: |
+ # docker build --file .docker/Dockerfile-unique \
+ # --build-arg RUNTIME_FEATURES=${{ matrix.runtime_features }} \
+ # --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} \
+ # --tag uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-${{ env.BUILD_SHA }} \
+ # .
- - name: Build the stack
- run: docker-compose -f ".docker/docker-compose.collators.${{ matrix.network }}.yml" up -d --remove-orphans --force-recreate --timeout 300
+ # - name: Push docker image version
+ # run: docker push uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-${{ env.BUILD_SHA }}
- # 🚀 POLKADOT LAUNCH COMPLETE 🚀
- - name: Check if docker logs consist messages related to testing of collator tests
- if: success()
- run: |
- counter=160
- function check_container_status {
- docker inspect -f {{.State.Running}} collator-selection-${{ matrix.network }}
- }
- function do_docker_logs {
- docker logs --details collator-selection-${{ matrix.network }} 2>&1
- }
- function is_started {
- if [ "$(check_container_status)" == "true" ]; then
- echo "Container: collator-selection-${{ matrix.network }} RUNNING";
- echo "Check Docker logs"
- DOCKER_LOGS=$(do_docker_logs)
- if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then
- echo "🚀 POLKADOT LAUNCH COMPLETE 🚀"
- return 0
- else
- echo "Message not found in logs output, repeating..."
- return 1
- fi
- else
- echo "Container collator-selection-${{ matrix.network }} NOT RUNNING"
- echo "Halting all future checks"
- exit 1
- fi
- echo "something goes wrong"
- exit 1
- }
- while ! is_started; do
- echo "Waiting for special message in log files "
- sleep 30s
- counter=$(( $counter - 1 ))
- echo "Counter: $counter"
- if [ "$counter" -gt "0" ]; then
- continue
- else
- break
- fi
- done
- echo "Halting script"
- exit 0
- shell: bash
- - uses: actions/setup-node@v3
+ - name: Prepare latest
+ uses: ./.github/actions/buildContainer
+ id: latest
with:
+ container: uniquenetwork/ci-collator-selection-local
+ tag: ${{ matrix.network }}-${{ env.REF_SLUG }}-${{ env.BUILD_SHA }}
+ context: .
+ dockerfile: .docker/Dockerfile-unique
+ args: |
+ --build-arg RUNTIME_FEATURES=${{ matrix.runtime_features }}
+ --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}
+ dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}
+ dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }}
+
+ - uses: actions/setup-node@v3.5.1
+ with:
node-version: 16
- - name: Run Collator tests
+ - name: Install baedeker
+ uses: UniqueNetwork/baedeker-action/setup@built
+
+ - name: Setup library
+ run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library
+
+ - name: Start network
+ uses: UniqueNetwork/baedeker-action@built
+ id: bdk
+ with:
+ jpath: |
+ .baedeker/vendor
+ tla-str: |
+ relay_spec=westend-local
+ inputs: |
+ .baedeker/collator-selection.jsonnet
+ snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.name }}'}})
+ snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.latest.outputs.name }}'}})
+
+ - name: Upload network config
+ uses: actions/upload-artifact@v3
+ with:
+ name: ${{ matrix.network }}-network-config
+ path: ${{ steps.bdk.outputs.composeProject }}
+ retention-days: 2
+
+ - name: Run tests
working-directory: tests
- if: success() || failure()
run: |
yarn install
yarn add mochawesome
- node scripts/readyness.js
+ # Wanted by both wait_for_first_block
+ # export RPC_URL="${RELAY_SAPPHIRE_HTTP_URL:-${RELAY_OPAL_HTTP_URL:-${RELAY_QUARTZ_HTTP_URL:-${RELAY_UNIQUE_HTTP_URL:-}}}}"
+ ./scripts/wait_for_first_block.sh
echo "Ready to start tests"
NOW=$(date +%s) && yarn testCollators --reporter mochawesome --reporter-options reportFilename=test-collators-${NOW}
env:
- RPC_URL: http://127.0.0.1:9944/
+ RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }}
- - name: Test Report Collator
- uses: phoenix-actions/test-reporting@v8
- id: test-report-collator
- if: success() || failure() # run this step even if previous step failed
+ - name: Test Report
+ uses: phoenix-actions/test-reporting@v10
+ id: test-report
+ if: success() || failure()
with:
- name: Report Collator tests results - ${{ matrix.network }} # Name of the check run which will be created
- path: tests/mochawesome-report/test-collators-*.json # Path to test results
+ name: Collator Selection Tests ${{ matrix.network }}
+ path: tests/mochawesome-report/test-collators-*.json
reporter: mochawesome-json
fail-on-error: 'false'
- - name: Stop running containers
- if: always() # run this step always
- run: docker-compose -f ".docker/docker-compose.collators.${{ matrix.network }}.yml" down
+ - name: Clean Workspace
+ if: always()
+ uses: AutoModality/action-clean@v1.1.0
- name: Remove builder cache
- if: always() # run this step always
+ if: always()
run: |
- docker builder prune -f -a
- docker system prune -f
-
+ docker system prune -a -f
.github/workflows/node-only-update.ymldiffbeforeafterboth--- a/.github/workflows/node-only-update.yml
+++ b/.github/workflows/node-only-update.yml
@@ -93,6 +93,7 @@
variables: |
RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}
POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}
+
- name: Prepare polkadot
uses: ./.github/actions/buildContainer
id: polkadot
tests/src/config.tsdiffbeforeafterboth--- a/tests/src/config.ts
+++ b/tests/src/config.ts
@@ -18,7 +18,7 @@
const config = {
relayUrl: process.env.RELAY_URL || 'ws://127.0.0.1:9844',
- substrateUrl: process.env.RELAY_OPAL_URL || process.env.RELAY_QUARTZ_URL || process.env.RELAY_UNIQUE_URL || 'ws://127.0.0.1:9944',
+ substrateUrl: process.env.RELAY_OPAL_URL || process.env.RELAY_QUARTZ_URL || process.env.RELAY_UNIQUE_URL || process.env.RELAY_SAPPHIRE_URL || 'ws://127.0.0.1:9944',
acalaUrl: process.env.RELAY_ACALA_URL || 'ws://127.0.0.1:9946',
karuraUrl: process.env.RELAY_KARURA_URL || 'ws://127.0.0.1:9946',
moonbeamUrl: process.env.RELAY_MOONBEAM_URL || 'ws://127.0.0.1:9947',