difftreelog
Merge pull request #666 from UniqueNetwork/rapid-parallel-start-v2-docs
in: master
Rapid parallel start v2 docs
13 files changed
.github/workflows/ci-develop.ymldiffbeforeafterboth--- a/.github/workflows/ci-develop.yml
+++ b/.github/workflows/ci-develop.yml
@@ -1,51 +1,55 @@
+# Workflow which controls starts nested workflows.
name: develop
+# Triger: PR at 'develop' branch with following types of events.
on:
pull_request:
branches: [ 'develop' ]
types: [ opened, reopened, synchronize, ready_for_review, converted_to_draft ]
+#Concurency group for control execution queue over github runners.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
+# List of a jobs included into Workflow.
jobs:
yarn-test-dev:
- if: github.event.pull_request.draft == false
+ if: github.event.pull_request.draft == false # Conditional check for draft per job.
uses: ./.github/workflows/dev-build-tests_v2.yml
unit-test:
- if: github.event.pull_request.draft == false
+ if: github.event.pull_request.draft == false # Conditional check for draft per job.
uses: ./.github/workflows/unit-test_v2.yml
canary:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'canary')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'canary')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/canary.yml
- secrets: inherit # pass all secrets
+ secrets: inherit # pass all secrets from initial workflow to nested
xcm:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'xcm')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'xcm')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/xcm.yml
- secrets: inherit # pass all secrets
+ secrets: inherit # pass all secrets from initial workflow to nested
forkless:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/forkless.yml
node-only-update:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/node-only-update_v2.yml
parallel_and_sequential_tests:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'para')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'para')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/node-only-update_v3.yml
codestyle:
- if: github.event.pull_request.draft == false
+ if: github.event.pull_request.draft == false # Conditional check for draft per job.
uses: ./.github/workflows/codestyle_v2.yml
yarn_eslint:
- if: github.event.pull_request.draft == false
+ if: github.event.pull_request.draft == false # Conditional check for draft per job.
uses: ./.github/workflows/test_codestyle_v2.yml
.github/workflows/ci-master.ymldiffbeforeafterboth--- a/.github/workflows/ci-master.yml
+++ b/.github/workflows/ci-master.yml
@@ -1,14 +1,18 @@
+# Workflow which controls starts nested workflows.
name: master
+# Triger: PR at 'master' branch with following types of events.
on:
pull_request:
branches: [ 'master' ]
types: [ opened, reopened, synchronize, ready_for_review ]
+#Concurency group for control execution queue over github runners.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
+# List of a jobs included into Workflow.
jobs:
unit-test:
@@ -22,11 +26,11 @@
canary:
uses: ./.github/workflows/canary.yml
- secrets: inherit # pass all secrets
+ secrets: inherit # pass all secrets from initial workflow to nested
xcm:
uses: ./.github/workflows/xcm.yml
- secrets: inherit # pass all secrets
+ secrets: inherit # pass all secrets from initial workflow to nested
codestyle:
- uses: ./.github/workflows/codestyle_v2.yml
\ No newline at end of file
+ uses: ./.github/workflows/codestyle_v2.yml
.github/workflows/codestyle_v2.ymldiffbeforeafterboth--- a/.github/workflows/codestyle_v2.yml
+++ b/.github/workflows/codestyle_v2.yml
@@ -1,5 +1,8 @@
+# Nested workflow for checks related to formatting Rust code
+
name: cargo fmt
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
.github/workflows/dev-build-tests_v2.ymldiffbeforeafterboth--- a/.github/workflows/dev-build-tests_v2.yml
+++ b/.github/workflows/dev-build-tests_v2.yml
@@ -1,8 +1,9 @@
+# Integration test in --dev mode
+# https://cryptousetech.atlassian.net/browse/CI-12
name: yarn test dev
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
- # Triggers the workflow on push or pull request events but only for the master branch
workflow_call:
@@ -101,4 +102,4 @@
run: |
docker builder prune -f -a
docker system prune -f
- docker image prune -f -a
\ No newline at end of file
+ docker image prune -f -a
.github/workflows/forkless-update-data_v2.ymldiffbeforeafterboth1# Controls when the action will run.2on:3 workflow_call:456#Define Workflow variables7env:8 REPO_URL: ${{ github.server_url }}/${{ github.repository }}910# A workflow run is made up of one or more jobs that can run sequentially or in parallel11jobs:1213 execution-marix:1415 name: execution matrix1617 runs-on: self-hosted-ci18 outputs:19 matrix: ${{ steps.create_matrix.outputs.matrix }}2021 steps:2223 - name: Clean Workspace24 uses: AutoModality/action-clean@v1.1.02526 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it27 - uses: actions/checkout@v328 with:29 ref: ${{ github.head_ref }} #Checking out head commit3031 - name: Read .env file32 uses: xom9ikk/dotenv@v1.0.23334 - name: Create Execution matrix35 uses: CertainLach/create-matrix-action@v336 id: create_matrix37 with:38 matrix: |39 network {opal}, runtime {opal}, features {opal-runtime}, mainnet_branch {${{ env.OPAL_MAINNET_TAG }}}, replica_from_address {${{ env.OPAL_REPLICA_FROM }}}40 network {quartz}, runtime {quartz}, features {quartz-runtime}, mainnet_branch {${{ env.QUARTZ_MAINNET_TAG }}}, replica_from_address {${{ env.QUARTZ_REPLICA_FROM }}}41 network {unique}, runtime {unique}, features {unique-runtime}, mainnet_branch {${{ env.UNIQUE_MAINNET_TAG }}}, replica_from_address {${{ env.UNIQUE_REPLICA_FROM }}}4243 forkless-update-data:44 needs: execution-marix45 # The type of runner that the job will run on46 runs-on: [self-hosted-ci,large]47 timeout-minutes: 138048 49 name: ${{ matrix.network }}50 strategy:51 matrix:52 include: ${{fromJson(needs.execution-marix.outputs.matrix)}}5354 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.555657 steps:58 - name: Clean Workspace59 uses: AutoModality/action-clean@v1.1.06061 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it62 - uses: actions/checkout@v363 with:64 ref: ${{ github.head_ref }} #Checking out head commit6566 - name: Read .env file67 uses: xom9ikk/dotenv@v1.0.26869 - name: Generate ENV related extend file for docker-compose70 uses: cuchi/jinja2-action@v1.2.071 with:72 template: .docker/docker-compose.tmp-forkless-data.j273 output_file: .docker/docker-compose.${{ matrix.network }}.yml74 variables: |75 REPO_URL=${{ github.server_url }}/${{ github.repository }}.git76 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}77 POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }}78 POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }}79 MAINNET_TAG=${{ matrix.mainnet_tag }}80 MAINNET_BRANCH=${{ matrix.mainnet_branch }}81 FEATURE=${{ matrix.features }}82 RUNTIME=${{ matrix.runtime }}83 BRANCH=${{ github.head_ref }}84 REPLICA_FROM=${{ matrix.replica_from_address }}8586 - name: Show build configuration87 run: cat .docker/docker-compose.${{ matrix.network }}.yml8889 - name: Generate launch-config-forkless-data.json90 uses: cuchi/jinja2-action@v1.2.091 with:92 template: .docker/forkless-config/launch-config-forkless-data.j293 output_file: .docker/launch-config-forkless-data.json94 variables: |95 FEATURE=${{ matrix.features }}96 RUNTIME=${{ matrix.runtime }}9798 - name: Show launch-config-forkless configuration99 run: cat .docker/launch-config-forkless-data.json100101102 - name: Build the stack103 run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" up -d --build --force-recreate --timeout 300104105 - name: Check if docker logs consist logs related to Runtime Upgrade testing.106 if: success()107 run: |108 counter=160109 function check_container_status {110 docker inspect -f {{.State.Running}} node-parachain111 }112 function do_docker_logs {113 docker logs --details node-parachain 2>&1114 }115 function is_started {116 if [ "$(check_container_status)" == "true" ]; then117 echo "Container: node-parachain RUNNING";118 echo "Check Docker logs"119 DOCKER_LOGS=$(do_docker_logs)120 if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then121 echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"122 return 0123 elif [[ ${DOCKER_LOGS} = *"🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧"* ]];then124 echo "🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧"125 return 1126 else127 echo "Message not found in logs output, repeating..."128 return 1129 fi130 else131 echo "Container node-parachain not RUNNING"132 echo "Halting all future checks"133 exit 1134 fi135 exit 0136 }137 while ! is_started; do138 echo "Waiting for special message in log files "139 sleep 30s140 counter=$(( $counter - 1 ))141 echo "Counter: $counter"142 if [ "$counter" -gt "0" ]; then143 continue144 else145 break146 fi147 done148 echo "Halting script"149 exit 0150 shell: bash151152 - name: Collect Docker Logs153 if: success() || failure()154 uses: jwalton/gh-docker-logs@v2.2.0155 with:156 dest: './forkless-parachain-upgrade-data-logs.${{ matrix.features }}'157 images: 'node-parachain'158159 - name: Show Docker logs160 if: success() || failure()161 run: cat './forkless-parachain-upgrade-data-logs.${{ matrix.features }}/node-parachain.log'162163 - name: Stop running containers164 if: always() # run this step always165 run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down --volumes1# Forkless update with data replication2# https://cryptousetech.atlassian.net/browse/CI-4234# Triger: only call from main workflow(re-usable workflows)5on:6 workflow_call:789# A workflow run is made up of one or more jobs that can run sequentially or in parallel10jobs:1112 execution-marix:1314 name: execution matrix1516 runs-on: self-hosted-ci17 outputs:18 matrix: ${{ steps.create_matrix.outputs.matrix }}1920 steps:2122 - name: Clean Workspace23 uses: AutoModality/action-clean@v1.1.02425 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it26 - uses: actions/checkout@v327 with:28 ref: ${{ github.head_ref }} #Checking out head commit2930 - name: Read .env file31 uses: xom9ikk/dotenv@v1.0.23233 - name: Create Execution matrix34 uses: CertainLach/create-matrix-action@v335 id: create_matrix36 with:37 matrix: |38 network {opal}, runtime {opal}, features {opal-runtime}, mainnet_branch {${{ env.OPAL_MAINNET_TAG }}}, replica_from_address {${{ env.OPAL_REPLICA_FROM }}}39 network {quartz}, runtime {quartz}, features {quartz-runtime}, mainnet_branch {${{ env.QUARTZ_MAINNET_TAG }}}, replica_from_address {${{ env.QUARTZ_REPLICA_FROM }}}40 network {unique}, runtime {unique}, features {unique-runtime}, mainnet_branch {${{ env.UNIQUE_MAINNET_TAG }}}, replica_from_address {${{ env.UNIQUE_REPLICA_FROM }}}4142 forkless-update-data:43 needs: execution-marix44 # The type of runner that the job will run on45 runs-on: [self-hosted-ci,large]46 timeout-minutes: 138047 48 name: ${{ matrix.network }}49 strategy:50 matrix:51 include: ${{fromJson(needs.execution-marix.outputs.matrix)}}5253 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.545556 steps:57 - name: Clean Workspace58 uses: AutoModality/action-clean@v1.1.05960 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it61 - uses: actions/checkout@v362 with:63 ref: ${{ github.head_ref }} #Checking out head commit6465 - name: Read .env file66 uses: xom9ikk/dotenv@v1.0.26768 - name: Generate ENV related extend file for docker-compose69 uses: cuchi/jinja2-action@v1.2.070 with:71 template: .docker/docker-compose.tmp-forkless-data.j272 output_file: .docker/docker-compose.${{ matrix.network }}.yml73 variables: |74 REPO_URL=${{ github.server_url }}/${{ github.repository }}.git75 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}76 POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }}77 POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }}78 MAINNET_TAG=${{ matrix.mainnet_tag }}79 MAINNET_BRANCH=${{ matrix.mainnet_branch }}80 FEATURE=${{ matrix.features }}81 RUNTIME=${{ matrix.runtime }}82 BRANCH=${{ github.head_ref }}83 REPLICA_FROM=${{ matrix.replica_from_address }}8485 - name: Show build configuration86 run: cat .docker/docker-compose.${{ matrix.network }}.yml8788 - name: Generate launch-config-forkless-data.json89 uses: cuchi/jinja2-action@v1.2.090 with:91 template: .docker/forkless-config/launch-config-forkless-data.j292 output_file: .docker/launch-config-forkless-data.json93 variables: |94 FEATURE=${{ matrix.features }}95 RUNTIME=${{ matrix.runtime }}9697 - name: Show launch-config-forkless configuration98 run: cat .docker/launch-config-forkless-data.json99100101 - name: Build the stack102 run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" up -d --build --force-recreate --timeout 300103104 - name: Check if docker logs consist logs related to Runtime Upgrade testing.105 if: success()106 run: |107 counter=160108 function check_container_status {109 docker inspect -f {{.State.Running}} node-parachain110 }111 function do_docker_logs {112 docker logs --details node-parachain 2>&1113 }114 function is_started {115 if [ "$(check_container_status)" == "true" ]; then116 echo "Container: node-parachain RUNNING";117 echo "Check Docker logs"118 DOCKER_LOGS=$(do_docker_logs)119 if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then120 echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"121 return 0122 elif [[ ${DOCKER_LOGS} = *"🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧"* ]];then123 echo "🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧"124 return 1125 else126 echo "Message not found in logs output, repeating..."127 return 1128 fi129 else130 echo "Container node-parachain not RUNNING"131 echo "Halting all future checks"132 exit 1133 fi134 exit 0135 }136 while ! is_started; do137 echo "Waiting for special message in log files "138 sleep 30s139 counter=$(( $counter - 1 ))140 echo "Counter: $counter"141 if [ "$counter" -gt "0" ]; then142 continue143 else144 break145 fi146 done147 echo "Halting script"148 exit 0149 shell: bash150151 - name: Collect Docker Logs152 if: success() || failure()153 uses: jwalton/gh-docker-logs@v2.2.0154 with:155 dest: './forkless-parachain-upgrade-data-logs.${{ matrix.features }}'156 images: 'node-parachain'157158 - name: Show Docker logs159 if: success() || failure()160 run: cat './forkless-parachain-upgrade-data-logs.${{ matrix.features }}/node-parachain.log'161162 - name: Stop running containers163 if: always() # run this step always164 run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down --volumes165 166 - name: Remove builder cache167 if: always() # run this step always168 run: |169 docker builder prune -f -a170 docker system prune -f171 docker image prune -f -a.github/workflows/forkless-update-nodata_v2.ymldiffbeforeafterboth--- a/.github/workflows/forkless-update-nodata_v2.yml
+++ b/.github/workflows/forkless-update-nodata_v2.yml
@@ -1,6 +1,7 @@
+# Forkless update without data replication
+# https://cryptousetech.atlassian.net/browse/CI-41
-
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
@@ -213,3 +214,10 @@
- name: Stop running containers
if: always() # run this step always
run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down
+
+ - name: Remove builder cache
+ if: always() # run this step always
+ run: |
+ docker builder prune -f -a
+ docker system prune -f
+ docker image prune -f -a
.github/workflows/forkless.ymldiffbeforeafterboth--- a/.github/workflows/forkless.yml
+++ b/.github/workflows/forkless.yml
@@ -1,3 +1,5 @@
+# Intermediate Nested Workflow for calling subworkflows as a parallel tasks.
+
name: Nesting Forkless
on:
.github/workflows/market-test_v2.ymldiffbeforeafterboth--- a/.github/workflows/market-test_v2.yml
+++ b/.github/workflows/market-test_v2.yml
@@ -1,3 +1,6 @@
+#https://cryptousetech.atlassian.net/browse/CI-37
+# Nested workflow for lunching Market e2e tests from external repository https://github.com/UniqueNetwork/market-e2e-tests
+
name: market api tests
# Controls when the action will run.
@@ -22,14 +25,8 @@
include:
- network: "opal"
features: "opal-runtime"
-# - network: "quartz"
-# features: "quartz-runtime"
-# - network: "unique"
-# features: "unique-runtime"
steps:
-
-
- name: Clean Workspace
uses: AutoModality/action-clean@v1.1.0
@@ -188,10 +185,3 @@
run: |
docker builder prune -f
docker system prune -f
-
- - name: Clean Workspace
- if: always()
- uses: AutoModality/action-clean@v1.1.0
-
-
-
.github/workflows/node-only-update_v2.ymldiffbeforeafterboth--- a/.github/workflows/node-only-update_v2.yml
+++ b/.github/workflows/node-only-update_v2.yml
@@ -1,12 +1,13 @@
+# https://cryptousetech.atlassian.net/browse/CI-85
+# Node only update with restart polkadot-launch process.
+
name: nodes-only update
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
-#Define Workflow variables
-env:
- REPO_URL: ${{ github.server_url }}/${{ github.repository }}
+
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
.github/workflows/node-only-update_v3.ymldiffbeforeafterboth--- a/.github/workflows/node-only-update_v3.yml
+++ b/.github/workflows/node-only-update_v3.yml
@@ -1,6 +1,7 @@
+# Test workflow for debug parallel and sequental tests in scope of execution time reducing.
name: Parallele tests
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
#Define Workflow variables
.github/workflows/test_codestyle_v2.ymldiffbeforeafterboth--- a/.github/workflows/test_codestyle_v2.yml
+++ b/.github/workflows/test_codestyle_v2.yml
@@ -1,5 +1,8 @@
+# Yarn Eslint over tests
+#
name: yarn eslint
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
.github/workflows/try-runtime_v2.ymldiffbeforeafterboth--- a/.github/workflows/try-runtime_v2.yml
+++ b/.github/workflows/try-runtime_v2.yml
@@ -1,3 +1,7 @@
+# Try-runtime checks
+# https://cryptousetech.atlassian.net/browse/CI-38
+
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
@@ -66,3 +70,10 @@
- name: Stop running containers
if: always() # run this step always
run: docker-compose -f ".docker/docker-compose-try-runtime.yml" -f ".docker/docker-compose.try-runtime.${{ matrix.network }}.yml" down
+
+ - name: Remove builder cache
+ if: always() # run this step always
+ run: |
+ docker builder prune -f -a
+ docker system prune -f
+ docker image prune -f -a
.github/workflows/unit-test_v2.ymldiffbeforeafterboth--- a/.github/workflows/unit-test_v2.yml
+++ b/.github/workflows/unit-test_v2.yml
@@ -1,8 +1,9 @@
+# Re-Usable Workflow for lanching Unit tests
name: unit tests
# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
- # Triggers the workflow on push or pull request events but only for the master branch
workflow_call:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
@@ -49,3 +50,10 @@
- name: Stop running containers
if: always() # run this step always
run: docker-compose -f ".docker/docker-compose-dev.yaml" -f ".docker/docker-compose.unit.yml" down
+
+ - name: Remove builder cache
+ if: always() # run this step always
+ run: |
+ docker builder prune -f -a
+ docker system prune -f
+ docker image prune -f -a