difftreelog
ci move collator selection to baedeker (#979)
in: master
* modify collator selection workflow for baedeker * fix collator selection workflow for baedeker * add modify in spec for collator selection workflow * test: also accept sapphire as substrateUrl * modify workflow collator-selection for baedeker * ci: remove unused collator-selection setups ---------
4 files changed
.baedeker/collator-selection.jsonnetdiffbeforeafterboth--- /dev/null
+++ b/.baedeker/collator-selection.jsonnet
@@ -0,0 +1,43 @@
+local
+m = import 'baedeker-library/mixin/spec.libsonnet',
+;
+
+local relay = {
+ name: 'relay',
+ bin: 'bin/polkadot',
+ validatorIdAssignment: 'staking',
+ spec: {Genesis:{
+ chain: 'rococo-local',
+ modify:: m.genericRelay($),
+ }},
+ nodes: {
+ [name]: {
+ bin: $.bin,
+ wantedKeys: 'relay',
+ },
+ for name in ['alice', 'bob', 'charlie', 'dave', 'eve']
+ },
+};
+
+local unique = {
+ name: 'unique',
+ bin: 'bin/unique',
+ paraId: 1001,
+ spec: {Genesis:{
+ modify:: m.genericPara($),
+ }},
+ nodes: {
+ [name]: {
+ bin: $.bin,
+ wantedKeys: 'para',
+ },
+ for name in ['alice', 'bob']
+ },
+};
+
+relay + {
+ parachains: {
+ [para.name]: para,
+ for para in [unique]
+ },
+}
.github/workflows/collator-selection.ymldiffbeforeafterboth1name: collator-selection-test-run1name: collator-selection-test-run223# Triger: only call from main workflow(re-usable workflows)3# Controls when the action will run.4on:4on:5 workflow_call:5 workflow_call:667# A workflow run is made up of one or more jobs that can run sequentially or in parallel7 # Allows you to run this workflow manually from the Actions tab8 workflow_dispatch:910#Define Workflow variables11env:12 REPO_URL: ${{ github.server_url }}/${{ github.repository }}1314# A workflow run is made up of one or more jobs that can run sequentially or in parallel8jobs:15jobs:169 prepare-execution-marix:17 prepare-execution-marix:1810 name: Prepare execution matrix19 name: Prepare execution matrix112012 runs-on: self-hosted-ci21 runs-on: [self-hosted-ci]13 outputs:22 outputs:14 matrix: ${{ steps.create_matrix.outputs.matrix }}23 matrix: ${{ steps.create_matrix.outputs.matrix }}152419 uses: AutoModality/action-clean@v1.1.028 uses: AutoModality/action-clean@v1.1.0202921 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it30 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it22 - uses: actions/checkout@v331 - uses: actions/checkout@v3.1.023 with:32 with:24 ref: ${{ github.head_ref }} #Checking out head commit33 ref: ${{ github.head_ref }} #Checking out head commit253426 - name: Read .env file35 - name: Read .env file27 uses: xom9ikk/dotenv@v236 uses: xom9ikk/dotenv@v2283729 - name: Create Execution matrix38 - name: Create Execution matrix30 uses: CertainLach/create-matrix-action@v339 uses: CertainLach/create-matrix-action@v431 id: create_matrix40 id: create_matrix32 with:41 with:33 matrix: |42 matrix: |34 network {opal}, wasm_name {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}, replica_from_address {${{ env.OPAL_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}43 network {opal}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}35 network {quartz}, wasm_name {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, replica_from_address {${{ env.QUARTZ_REPLICA_FROM }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}44 network {quartz}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}36 network {sapphire}, wasm_name {quartz}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, replica_from_address {${{ env.SAPPHIRE_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}45 network {sapphire}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}3738 collator-selection-build:39 needs: prepare-execution-marix40 # The type of runner that the job will run on41 runs-on: [self-hosted-ci,large]42 timeout-minutes: 13804344 name: ${{ matrix.network }}-build45 strategy:46 matrix:47 include: ${{fromJson(needs.prepare-execution-marix.outputs.matrix)}}4849 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.5051 steps:52 - name: Clean Workspace53 uses: AutoModality/action-clean@v1.1.05455 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it56 - uses: actions/checkout@v357 with:58 ref: ${{ github.head_ref }} #Checking out head commit5960 - name: Read .env file61 uses: xom9ikk/dotenv@v26263 # Prepare SHA 64 - name: Prepare SHA65 uses: ./.github/actions/prepare6667 # Build main image for FORKLESS-UPDATE-NODATA68 - name: Generate ENV related extend Dockerfile file69 uses: cuchi/jinja2-action@v1.2.070 with:71 template: .docker/Dockerfile-collators.j272 output_file: .docker/Dockerfile-collators.${{ matrix.network }}.yml73 variables: |74 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}75 NETWORK=${{ matrix.network }}76 POLKADOT_LAUNCH_BRANCH=${{ env.POLKADOT_LAUNCH_BRANCH }}77 POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}78 BRANCH=${{ github.head_ref }}7980 - name: Show build configuration81 run: cat .docker/Dockerfile-collators.${{ matrix.network }}.yml8283 - name: Run find-and-replace to remove slashes from branch name84 uses: mad9000/actions-find-and-replace-string@485 id: branchname86 with:87 source: ${{ github.head_ref }}88 find: '/'89 replace: '-'9091 - name: Set build SHA92 shell: bash93 run: |94 echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV9596 - name: Build the stack97 run: cd .docker/ && docker build --no-cache --file ./Dockerfile-collators.${{ matrix.network }}.yml --tag uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA ../9899 - name: Log in to Docker Hub100 uses: docker/login-action@v2.1.0101 with:102 username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}103 password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}104105 - name: Push docker image version106 run: docker push uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA107108 - name: Remove builder cache109 if: always() # run this step always110 run: |111 docker builder prune -f112 docker system prune -f11346114 collator-selection-tests:47 collator-selection:48115 needs: [prepare-execution-marix, collator-selection-build]49 needs: prepare-execution-marix116 # The type of runner that the job will run on50 # The type of runner that the job will run on117 runs-on: [self-hosted-ci, large]51 runs-on: [self-hosted-ci,large]11852119 timeout-minutes: 60053 timeout-minutes: 138012054121 name: ${{ matrix.network }}-tests55 name: ${{ matrix.network }}12256123 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.57 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.12458139 with:73 with:140 ref: ${{ github.head_ref }} #Checking out head commit74 ref: ${{ github.head_ref }} #Checking out head commit1417576 # Prepare SHA142 - name: Prepare77 - name: Prepare SHA143 uses: ./.github/actions/prepare78 uses: ./.github/actions/prepare7980 - name: Read .env file81 uses: xom9ikk/dotenv@v214482145 - name: Set build SHA83 - name: Log in to Docker Hub146 shell: bash84 uses: docker/login-action@v2.1.085 with:86 username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}147 run: |87 password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}148 echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV14988150 - name: Run find-and-replace to remove slashes from branch name89 - name: Run find-and-replace to remove slashes from branch name151 uses: mad9000/actions-find-and-replace-string@490 uses: mad9000/actions-find-and-replace-string@4155 find: '/'94 find: '/'156 replace: '-'95 replace: '-'157158 - name: Read .env file159 uses: xom9ikk/dotenv@v216096161 - name: Generate ENV related extend file for docker-compose97 - name: Generate ENV related extend Dockerfile file for POLKADOT162 uses: cuchi/jinja2-action@v1.2.098 uses: cuchi/jinja2-action@v1.2.0163 with:99 with:164 template: .docker/docker-compose.collators.j2100 template: .docker/Dockerfile-polkadot.j2165 output_file: .docker/docker-compose.collators.${{ matrix.network }}.yml101 output_file: .docker/Dockerfile-polkadot.${{ matrix.relay_branch }}.yml166 variables: |102 variables: |167 NETWORK=${{ matrix.network }}103 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}168 BUILD_TAG=${{ steps.branchname.outputs.value }}-$BUILD_SHA104 POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}169105 170 - name: Show build configuration106 - name: Prepare polkadot107 uses: ./.github/actions/buildContainer108 id: polkadot109 with:110 container: uniquenetwork/builder-polkadot111 tag: ${{ matrix.relay_branch }}112 context: .docker171 run: cat .docker/docker-compose.collators.${{ matrix.network }}.yml113 dockerfile: Dockerfile-polkadot.${{ matrix.relay_branch }}.yml114 dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}115 dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }}116117 # - name: pull needed images118 # run: |119 # docker pull uniquenetwork/builder-polkadot:${{ matrix.relay_branch }}120121 # - name: Build unique-chain122 # run: |123 # docker build --file .docker/Dockerfile-unique \124 # --build-arg RUNTIME_FEATURES=${{ matrix.runtime_features }} \125 # --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} \126 # --tag uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-${{ env.BUILD_SHA }} \127 # .128129 # - name: Push docker image version130 # run: docker push uniquenetwork/ci-collator-selection-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-${{ env.BUILD_SHA }}131172132173 - name: Log in to Docker Hub133 - name: Prepare latest174 uses: docker/login-action@v2.1.0134 uses: ./.github/actions/buildContainer135 id: latest175 with:136 with:137 container: uniquenetwork/ci-collator-selection-local138 tag: ${{ matrix.network }}-${{ env.REF_SLUG }}-${{ env.BUILD_SHA }}139 context: .140 dockerfile: .docker/Dockerfile-unique141 args: |142 --build-arg RUNTIME_FEATURES=${{ matrix.runtime_features }}143 --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}176 username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}144 dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }}177 password: ${{ secrets.CORE_DOCKERHUB_TOKEN }}145 dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }}146147 - uses: actions/setup-node@v3.5.1148 with:149 node-version: 16150151 - name: Install baedeker152 uses: UniqueNetwork/baedeker-action/setup@built178153179 - name: Build the stack154 - name: Setup library180 run: docker-compose -f ".docker/docker-compose.collators.${{ matrix.network }}.yml" up -d --remove-orphans --force-recreate --timeout 300155 run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library181156182 # 🚀 POLKADOT LAUNCH COMPLETE 🚀183 - name: Check if docker logs consist messages related to testing of collator tests 157 - name: Start network158 uses: UniqueNetwork/baedeker-action@built184 if: success()159 id: bdk160 with:185 run: |161 jpath: |186 counter=160162 .baedeker/vendor187 function check_container_status {188 docker inspect -f {{.State.Running}} collator-selection-${{ matrix.network }}189 }190 function do_docker_logs {191 docker logs --details collator-selection-${{ matrix.network }} 2>&1192 }193 function is_started {194 if [ "$(check_container_status)" == "true" ]; then195 echo "Container: collator-selection-${{ matrix.network }} RUNNING";196 echo "Check Docker logs"197 DOCKER_LOGS=$(do_docker_logs)198 if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then199 echo "🚀 POLKADOT LAUNCH COMPLETE 🚀"200 return 0201 else202 echo "Message not found in logs output, repeating..."203 return 1204 fi205 else206 echo "Container collator-selection-${{ matrix.network }} NOT RUNNING"207 echo "Halting all future checks"208 exit 1209 fi210 echo "something goes wrong"211 exit 1212 }213 while ! is_started; do214 echo "Waiting for special message in log files "215 sleep 30s216 counter=$(( $counter - 1 ))217 echo "Counter: $counter"218 if [ "$counter" -gt "0" ]; then219 continue220 else221 break222 fi223 done224 echo "Halting script"225 exit 0226 shell: bash163 tla-str: |164 relay_spec=westend-local 165 inputs: |166 .baedeker/collator-selection.jsonnet167 snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.name }}'}})168 snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.latest.outputs.name }}'}})227169228 - uses: actions/setup-node@v3170 - name: Upload network config171 uses: actions/upload-artifact@v3229 with:172 with:173 name: ${{ matrix.network }}-network-config230 node-version: 16174 path: ${{ steps.bdk.outputs.composeProject }}175 retention-days: 2231176232 - name: Run Collator tests177 - name: Run tests233 working-directory: tests178 working-directory: tests234 if: success() || failure()235 run: |179 run: |236 yarn install180 yarn install237 yarn add mochawesome181 yarn add mochawesome238 node scripts/readyness.js182 # Wanted by both wait_for_first_block183 # export RPC_URL="${RELAY_SAPPHIRE_HTTP_URL:-${RELAY_OPAL_HTTP_URL:-${RELAY_QUARTZ_HTTP_URL:-${RELAY_UNIQUE_HTTP_URL:-}}}}"184 ./scripts/wait_for_first_block.sh239 echo "Ready to start tests"185 echo "Ready to start tests"240 NOW=$(date +%s) && yarn testCollators --reporter mochawesome --reporter-options reportFilename=test-collators-${NOW}186 NOW=$(date +%s) && yarn testCollators --reporter mochawesome --reporter-options reportFilename=test-collators-${NOW}241 env:187 env:242 RPC_URL: http://127.0.0.1:9944/188 RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} 243189244 - name: Test Report Collator190 - name: Test Report245 uses: phoenix-actions/test-reporting@v8191 uses: phoenix-actions/test-reporting@v10246 id: test-report-collator192 id: test-report247 if: success() || failure() # run this step even if previous step failed193 if: success() || failure()248 with:194 with:249 name: Report Collator tests results - ${{ matrix.network }} # Name of the check run which will be created195 name: Collator Selection Tests ${{ matrix.network }}250 path: tests/mochawesome-report/test-collators-*.json # Path to test results196 path: tests/mochawesome-report/test-collators-*.json251 reporter: mochawesome-json197 reporter: mochawesome-json252 fail-on-error: 'false'198 fail-on-error: 'false'253199254 - name: Stop running containers200 - name: Clean Workspace255 if: always() # run this step always201 if: always()256 run: docker-compose -f ".docker/docker-compose.collators.${{ matrix.network }}.yml" down202 uses: AutoModality/action-clean@v1.1.0257203258 - name: Remove builder cache204 - name: Remove builder cache259 if: always() # run this step always205 if: always()260 run: |206 run: |261 docker builder prune -f -a207 docker system prune -a -f262 docker system prune -f263208264.github/workflows/node-only-update.ymldiffbeforeafterboth--- a/.github/workflows/node-only-update.yml
+++ b/.github/workflows/node-only-update.yml
@@ -93,6 +93,7 @@
variables: |
RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}
POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }}
+
- name: Prepare polkadot
uses: ./.github/actions/buildContainer
id: polkadot
tests/src/config.tsdiffbeforeafterboth--- a/tests/src/config.ts
+++ b/tests/src/config.ts
@@ -18,7 +18,7 @@
const config = {
relayUrl: process.env.RELAY_URL || 'ws://127.0.0.1:9844',
- substrateUrl: process.env.RELAY_OPAL_URL || process.env.RELAY_QUARTZ_URL || process.env.RELAY_UNIQUE_URL || 'ws://127.0.0.1:9944',
+ substrateUrl: process.env.RELAY_OPAL_URL || process.env.RELAY_QUARTZ_URL || process.env.RELAY_UNIQUE_URL || process.env.RELAY_SAPPHIRE_URL || 'ws://127.0.0.1:9944',
acalaUrl: process.env.RELAY_ACALA_URL || 'ws://127.0.0.1:9946',
karuraUrl: process.env.RELAY_KARURA_URL || 'ws://127.0.0.1:9946',
moonbeamUrl: process.env.RELAY_MOONBEAM_URL || 'ws://127.0.0.1:9947',