difftreelog
Merge pull request #666 from UniqueNetwork/rapid-parallel-start-v2-docs
in: master
Rapid parallel start v2 docs
13 files changed
.github/workflows/ci-develop.ymldiffbeforeafterboth--- a/.github/workflows/ci-develop.yml
+++ b/.github/workflows/ci-develop.yml
@@ -1,51 +1,55 @@
+# Workflow which controls starts nested workflows.
name: develop
+# Triger: PR at 'develop' branch with following types of events.
on:
pull_request:
branches: [ 'develop' ]
types: [ opened, reopened, synchronize, ready_for_review, converted_to_draft ]
+#Concurency group for control execution queue over github runners.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
+# List of a jobs included into Workflow.
jobs:
yarn-test-dev:
- if: github.event.pull_request.draft == false
+ if: github.event.pull_request.draft == false # Conditional check for draft per job.
uses: ./.github/workflows/dev-build-tests_v2.yml
unit-test:
- if: github.event.pull_request.draft == false
+ if: github.event.pull_request.draft == false # Conditional check for draft per job.
uses: ./.github/workflows/unit-test_v2.yml
canary:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'canary')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'canary')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/canary.yml
- secrets: inherit # pass all secrets
+ secrets: inherit # pass all secrets from initial workflow to nested
xcm:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'xcm')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'xcm')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/xcm.yml
- secrets: inherit # pass all secrets
+ secrets: inherit # pass all secrets from initial workflow to nested
forkless:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/forkless.yml
node-only-update:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/node-only-update_v2.yml
parallel_and_sequential_tests:
- if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'para')) }}
+ if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'para')) }} # Conditional check for draft & labels per job.
uses: ./.github/workflows/node-only-update_v3.yml
codestyle:
- if: github.event.pull_request.draft == false
+ if: github.event.pull_request.draft == false # Conditional check for draft per job.
uses: ./.github/workflows/codestyle_v2.yml
yarn_eslint:
- if: github.event.pull_request.draft == false
+ if: github.event.pull_request.draft == false # Conditional check for draft per job.
uses: ./.github/workflows/test_codestyle_v2.yml
.github/workflows/ci-master.ymldiffbeforeafterboth--- a/.github/workflows/ci-master.yml
+++ b/.github/workflows/ci-master.yml
@@ -1,14 +1,18 @@
+# Workflow which controls starts nested workflows.
name: master
+# Triger: PR at 'master' branch with following types of events.
on:
pull_request:
branches: [ 'master' ]
types: [ opened, reopened, synchronize, ready_for_review ]
+#Concurency group for control execution queue over github runners.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
+# List of a jobs included into Workflow.
jobs:
unit-test:
@@ -22,11 +26,11 @@
canary:
uses: ./.github/workflows/canary.yml
- secrets: inherit # pass all secrets
+ secrets: inherit # pass all secrets from initial workflow to nested
xcm:
uses: ./.github/workflows/xcm.yml
- secrets: inherit # pass all secrets
+ secrets: inherit # pass all secrets from initial workflow to nested
codestyle:
- uses: ./.github/workflows/codestyle_v2.yml
\ No newline at end of file
+ uses: ./.github/workflows/codestyle_v2.yml
.github/workflows/codestyle_v2.ymldiffbeforeafterboth--- a/.github/workflows/codestyle_v2.yml
+++ b/.github/workflows/codestyle_v2.yml
@@ -1,5 +1,8 @@
+# Nested workflow for checks related to formatting Rust code
+
name: cargo fmt
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
.github/workflows/dev-build-tests_v2.ymldiffbeforeafterboth--- a/.github/workflows/dev-build-tests_v2.yml
+++ b/.github/workflows/dev-build-tests_v2.yml
@@ -1,8 +1,9 @@
+# Integration test in --dev mode
+# https://cryptousetech.atlassian.net/browse/CI-12
name: yarn test dev
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
- # Triggers the workflow on push or pull request events but only for the master branch
workflow_call:
@@ -101,4 +102,4 @@
run: |
docker builder prune -f -a
docker system prune -f
- docker image prune -f -a
\ No newline at end of file
+ docker image prune -f -a
.github/workflows/forkless-update-data_v2.ymldiffbeforeafterboth--- a/.github/workflows/forkless-update-data_v2.yml
+++ b/.github/workflows/forkless-update-data_v2.yml
@@ -1,11 +1,10 @@
-# Controls when the action will run.
+# Forkless update with data replication
+# https://cryptousetech.atlassian.net/browse/CI-42
+
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
-
-#Define Workflow variables
-env:
- REPO_URL: ${{ github.server_url }}/${{ github.repository }}
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
@@ -163,3 +162,10 @@
- name: Stop running containers
if: always() # run this step always
run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down --volumes
+
+ - name: Remove builder cache
+ if: always() # run this step always
+ run: |
+ docker builder prune -f -a
+ docker system prune -f
+ docker image prune -f -a
.github/workflows/forkless-update-nodata_v2.ymldiffbeforeafterboth--- a/.github/workflows/forkless-update-nodata_v2.yml
+++ b/.github/workflows/forkless-update-nodata_v2.yml
@@ -1,6 +1,7 @@
+# Forkless update without data replication
+# https://cryptousetech.atlassian.net/browse/CI-41
-
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
@@ -213,3 +214,10 @@
- name: Stop running containers
if: always() # run this step always
run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down
+
+ - name: Remove builder cache
+ if: always() # run this step always
+ run: |
+ docker builder prune -f -a
+ docker system prune -f
+ docker image prune -f -a
.github/workflows/forkless.ymldiffbeforeafterboth--- a/.github/workflows/forkless.yml
+++ b/.github/workflows/forkless.yml
@@ -1,3 +1,5 @@
+# Intermediate Nested Workflow for calling subworkflows as a parallel tasks.
+
name: Nesting Forkless
on:
.github/workflows/market-test_v2.ymldiffbeforeafterboth1name: market api tests23# Controls when the action will run.4on:5 workflow_call:678# A workflow run is made up of one or more jobs that can run sequentially or in parallel9jobs:1011 market_test:12 # The type of runner that the job will run on13 runs-on: [self-hosted-ci,large]14 timeout-minutes: 3601516 name: ${{ matrix.network }} 1718 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.19 20 strategy:21 matrix:22 include:23 - network: "opal"24 features: "opal-runtime"25# - network: "quartz"26# features: "quartz-runtime"27# - network: "unique"28# features: "unique-runtime"2930 steps:313233 - name: Clean Workspace34 uses: AutoModality/action-clean@v1.1.03536 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it37 - name: Checkout master repo38 uses: actions/checkout@master39 with:40 ref: ${{ github.head_ref }} #Checking out head commit4142 - name: Checkout Market e2e tests43 uses: actions/checkout@v344 with:45 repository: 'UniqueNetwork/market-e2e-tests'46 ssh-key: ${{ secrets.GH_PAT }}47 path: 'qa-tests'48 ref: 'master'4950 - name: Read .env file51 uses: xom9ikk/dotenv@v1.0.252 53 - name: Copy qa-tests/.env.example to qa-tests/.env54 working-directory: qa-tests55 run: cp .env.docker .env56 57 - name: Generate ENV related extend file for docker-compose58 uses: cuchi/jinja2-action@v1.2.059 with:60 template: qa-tests/.docker/docker-compose.tmp-market.j261 output_file: qa-tests/.docker/docker-compose.${{ matrix.network }}.yml62 variables: |63 REPO_URL=${{ github.server_url }}/${{ github.repository }}.git64 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}65 POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }}66 FEATURE=${{ matrix.features }}67 BRANCH=${{ github.head_ref }}686970 - name: Show build configuration71 working-directory: qa-tests72 run: cat .docker/docker-compose.${{ matrix.network }}.yml7374 - name: Start node-parachain75 working-directory: qa-tests76 run: docker-compose -f ".docker/docker-compose.market.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" up -d --build --remove-orphans --force-recreate node-parachain 77 78 - uses: actions/setup-node@v379 with:80 node-version: 16.178182 - name: Setup TypeScript83 working-directory: qa-tests84 run: |85 npm install -g ts-node86 npm install8788 - name: Copy qa-tests/.env.docker to qa-tests/.env89 working-directory: qa-tests90 run: | 91 rm -rf .env92 cp .env.docker .env9394# Temporary disable node readyness check. Have to dig into the script logic.95# - name: Wait for chain up and running96# working-directory: tests97# run: |98# yarn install99# node scripts/readyness.js100# echo "Ready to start tests"101# env:102# RPC_URL: http://127.0.0.1:9933/103104 - name: Wait for chain up and running105 run: |106 sleep 1200s107 echo "Ready to start Market e2e tests"108109 - name: Show content of .env file and Generate accounts110 working-directory: qa-tests111 run: |112 cat .env113 ts-node ./src/scripts/create-market-accounts.ts114115 - name: Copy qa-tests/.env to qa-tests/.env.docker116 working-directory: qa-tests117 run: | 118 rm -rf .env.docker119 cp .env .env.docker120121 - name: Get chain logs122 if: always() # run this step always123 run: |124 docker exec node-parachain cat /polkadot-launch/9944.log125 docker exec node-parachain cat /polkadot-launch/9945.log126 docker exec node-parachain cat /polkadot-launch/alice.log127 docker exec node-parachain cat /polkadot-launch/eve.log128 docker exec node-parachain cat /polkadot-launch/dave.log129 docker exec node-parachain cat /polkadot-launch/charlie.log130 131 - name: Deploy contracts132 run: |133 cd qa-tests134 ts-node ./src/scripts/deploy-contract.ts135136 - name: Timeout for debug137 if: failure()138 run: sleep 300s139140 - name: Import test data141 working-directory: qa-tests142 run: ts-node ./src/scripts/create-test-collections.ts143144 - name: Show content of qa-test .env145 working-directory: qa-tests146 run: cat .env147148 - name: Read qa -test .env file Before market start149 uses: xom9ikk/dotenv@v1.0.2150 with:151 path: qa-tests/152153 - name: local-market:start154 run: docker-compose -f "qa-tests/.docker/docker-compose.market.yml" -f "qa-tests/.docker/docker-compose.${{ matrix.network }}.yml" up -d --build155156 - name: Wait for market readyness157 working-directory: qa-tests158 run: src/scripts/wait-market-ready.sh159 shell: bash160161 - name: Install dependecies162 working-directory: qa-tests163 run: |164 npm ci165 npm install -D @playwright/test166 npx playwright install-deps167 npx playwright install168169 - name: Show content of qa-test .env170 working-directory: qa-tests171 run: cat .env172 173 - name: Test API interface174 working-directory: qa-tests175 run: |176 npx playwright test --workers=8 --quiet .*.api.test.ts --reporter=github --config playwright.config.ts177178 - name: Timeout for debug179 if: failure()180 run: sleep 300s181182 - name: Stop running containers183 if: always() # run this step always184 run: docker-compose -f "qa-tests/.docker/docker-compose.market.yml" -f "qa-tests/.docker/docker-compose.${{ matrix.network }}.yml" down --volumes185186 - name: Remove builder cache187 if: always() # run this step always188 run: |189 docker builder prune -f190 docker system prune -f191192 - name: Clean Workspace193 if: always()194 uses: AutoModality/action-clean@v1.1.01951961971#https://cryptousetech.atlassian.net/browse/CI-372# Nested workflow for lunching Market e2e tests from external repository https://github.com/UniqueNetwork/market-e2e-tests34name: market api tests56# Controls when the action will run.7on:8 workflow_call:91011# A workflow run is made up of one or more jobs that can run sequentially or in parallel12jobs:1314 market_test:15 # The type of runner that the job will run on16 runs-on: [self-hosted-ci,large]17 timeout-minutes: 3601819 name: ${{ matrix.network }} 2021 continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false.22 23 strategy:24 matrix:25 include:26 - network: "opal"27 features: "opal-runtime"2829 steps:30 - name: Clean Workspace31 uses: AutoModality/action-clean@v1.1.03233 # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it34 - name: Checkout master repo35 uses: actions/checkout@master36 with:37 ref: ${{ github.head_ref }} #Checking out head commit3839 - name: Checkout Market e2e tests40 uses: actions/checkout@v341 with:42 repository: 'UniqueNetwork/market-e2e-tests'43 ssh-key: ${{ secrets.GH_PAT }}44 path: 'qa-tests'45 ref: 'master'4647 - name: Read .env file48 uses: xom9ikk/dotenv@v1.0.249 50 - name: Copy qa-tests/.env.example to qa-tests/.env51 working-directory: qa-tests52 run: cp .env.docker .env53 54 - name: Generate ENV related extend file for docker-compose55 uses: cuchi/jinja2-action@v1.2.056 with:57 template: qa-tests/.docker/docker-compose.tmp-market.j258 output_file: qa-tests/.docker/docker-compose.${{ matrix.network }}.yml59 variables: |60 REPO_URL=${{ github.server_url }}/${{ github.repository }}.git61 RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}62 POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }}63 FEATURE=${{ matrix.features }}64 BRANCH=${{ github.head_ref }}656667 - name: Show build configuration68 working-directory: qa-tests69 run: cat .docker/docker-compose.${{ matrix.network }}.yml7071 - name: Start node-parachain72 working-directory: qa-tests73 run: docker-compose -f ".docker/docker-compose.market.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" up -d --build --remove-orphans --force-recreate node-parachain 74 75 - uses: actions/setup-node@v376 with:77 node-version: 16.177879 - name: Setup TypeScript80 working-directory: qa-tests81 run: |82 npm install -g ts-node83 npm install8485 - name: Copy qa-tests/.env.docker to qa-tests/.env86 working-directory: qa-tests87 run: | 88 rm -rf .env89 cp .env.docker .env9091# Temporary disable node readyness check. Have to dig into the script logic.92# - name: Wait for chain up and running93# working-directory: tests94# run: |95# yarn install96# node scripts/readyness.js97# echo "Ready to start tests"98# env:99# RPC_URL: http://127.0.0.1:9933/100101 - name: Wait for chain up and running102 run: |103 sleep 1200s104 echo "Ready to start Market e2e tests"105106 - name: Show content of .env file and Generate accounts107 working-directory: qa-tests108 run: |109 cat .env110 ts-node ./src/scripts/create-market-accounts.ts111112 - name: Copy qa-tests/.env to qa-tests/.env.docker113 working-directory: qa-tests114 run: | 115 rm -rf .env.docker116 cp .env .env.docker117118 - name: Get chain logs119 if: always() # run this step always120 run: |121 docker exec node-parachain cat /polkadot-launch/9944.log122 docker exec node-parachain cat /polkadot-launch/9945.log123 docker exec node-parachain cat /polkadot-launch/alice.log124 docker exec node-parachain cat /polkadot-launch/eve.log125 docker exec node-parachain cat /polkadot-launch/dave.log126 docker exec node-parachain cat /polkadot-launch/charlie.log127 128 - name: Deploy contracts129 run: |130 cd qa-tests131 ts-node ./src/scripts/deploy-contract.ts132133 - name: Timeout for debug134 if: failure()135 run: sleep 300s136137 - name: Import test data138 working-directory: qa-tests139 run: ts-node ./src/scripts/create-test-collections.ts140141 - name: Show content of qa-test .env142 working-directory: qa-tests143 run: cat .env144145 - name: Read qa -test .env file Before market start146 uses: xom9ikk/dotenv@v1.0.2147 with:148 path: qa-tests/149150 - name: local-market:start151 run: docker-compose -f "qa-tests/.docker/docker-compose.market.yml" -f "qa-tests/.docker/docker-compose.${{ matrix.network }}.yml" up -d --build152153 - name: Wait for market readyness154 working-directory: qa-tests155 run: src/scripts/wait-market-ready.sh156 shell: bash157158 - name: Install dependecies159 working-directory: qa-tests160 run: |161 npm ci162 npm install -D @playwright/test163 npx playwright install-deps164 npx playwright install165166 - name: Show content of qa-test .env167 working-directory: qa-tests168 run: cat .env169 170 - name: Test API interface171 working-directory: qa-tests172 run: |173 npx playwright test --workers=8 --quiet .*.api.test.ts --reporter=github --config playwright.config.ts174175 - name: Timeout for debug176 if: failure()177 run: sleep 300s178179 - name: Stop running containers180 if: always() # run this step always181 run: docker-compose -f "qa-tests/.docker/docker-compose.market.yml" -f "qa-tests/.docker/docker-compose.${{ matrix.network }}.yml" down --volumes182183 - name: Remove builder cache184 if: always() # run this step always185 run: |186 docker builder prune -f187 docker system prune -f.github/workflows/node-only-update_v2.ymldiffbeforeafterboth--- a/.github/workflows/node-only-update_v2.yml
+++ b/.github/workflows/node-only-update_v2.yml
@@ -1,12 +1,13 @@
+# https://cryptousetech.atlassian.net/browse/CI-85
+# Node only update with restart polkadot-launch process.
+
name: nodes-only update
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
-#Define Workflow variables
-env:
- REPO_URL: ${{ github.server_url }}/${{ github.repository }}
+
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
.github/workflows/node-only-update_v3.ymldiffbeforeafterboth--- a/.github/workflows/node-only-update_v3.yml
+++ b/.github/workflows/node-only-update_v3.yml
@@ -1,6 +1,7 @@
+# Test workflow for debug parallel and sequental tests in scope of execution time reducing.
name: Parallele tests
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
#Define Workflow variables
.github/workflows/test_codestyle_v2.ymldiffbeforeafterboth--- a/.github/workflows/test_codestyle_v2.yml
+++ b/.github/workflows/test_codestyle_v2.yml
@@ -1,5 +1,8 @@
+# Yarn Eslint over tests
+#
name: yarn eslint
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
.github/workflows/try-runtime_v2.ymldiffbeforeafterboth--- a/.github/workflows/try-runtime_v2.yml
+++ b/.github/workflows/try-runtime_v2.yml
@@ -1,3 +1,7 @@
+# Try-runtime checks
+# https://cryptousetech.atlassian.net/browse/CI-38
+
+# Triger: only call from main workflow(re-usable workflows)
on:
workflow_call:
@@ -66,3 +70,10 @@
- name: Stop running containers
if: always() # run this step always
run: docker-compose -f ".docker/docker-compose-try-runtime.yml" -f ".docker/docker-compose.try-runtime.${{ matrix.network }}.yml" down
+
+ - name: Remove builder cache
+ if: always() # run this step always
+ run: |
+ docker builder prune -f -a
+ docker system prune -f
+ docker image prune -f -a
.github/workflows/unit-test_v2.ymldiffbeforeafterboth--- a/.github/workflows/unit-test_v2.yml
+++ b/.github/workflows/unit-test_v2.yml
@@ -1,8 +1,9 @@
+# Re-Usable Workflow for lanching Unit tests
name: unit tests
# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
on:
- # Triggers the workflow on push or pull request events but only for the master branch
workflow_call:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
@@ -49,3 +50,10 @@
- name: Stop running containers
if: always() # run this step always
run: docker-compose -f ".docker/docker-compose-dev.yaml" -f ".docker/docker-compose.unit.yml" down
+
+ - name: Remove builder cache
+ if: always() # run this step always
+ run: |
+ docker builder prune -f -a
+ docker system prune -f
+ docker image prune -f -a