git.delta.rocks / unique-network / refs/commits / 3cddb472af6f

difftreelog

Merge pull request #666 from UniqueNetwork/rapid-parallel-start-v2-docs

Alex2022-10-19parents: #a63e642 #8d84680.patch.diff
in: master
Rapid parallel start v2 docs

13 files changed

modified.github/workflows/ci-develop.ymldiffbeforeafterboth
--- a/.github/workflows/ci-develop.yml
+++ b/.github/workflows/ci-develop.yml
@@ -1,51 +1,55 @@
+# Workflow which controls starts nested workflows.
 name: develop
 
+# Triger: PR at 'develop' branch with following types of events.
 on:
   pull_request:
     branches: [ 'develop' ]
     types: [ opened, reopened, synchronize, ready_for_review, converted_to_draft ]
 
+#Concurency group for control execution queue over github runners.
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+# List of a jobs included into Workflow.
 jobs:
 
   yarn-test-dev:
-    if: github.event.pull_request.draft == false
+    if: github.event.pull_request.draft == false                                                                             # Conditional check for draft per job.
     uses: ./.github/workflows/dev-build-tests_v2.yml
 
 
   unit-test:
-    if: github.event.pull_request.draft == false
+    if: github.event.pull_request.draft == false                                                                             # Conditional check for draft per job.
     uses: ./.github/workflows/unit-test_v2.yml
 
   canary:
-    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'canary')) }}
+    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'canary')) }}    # Conditional check for draft & labels per job.
     uses: ./.github/workflows/canary.yml
-    secrets: inherit # pass all secrets
+    secrets: inherit # pass all secrets from initial workflow to nested
 
   xcm:
-    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'xcm')) }}
+    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'xcm')) }}       # Conditional check for draft & labels per job.
     uses: ./.github/workflows/xcm.yml
-    secrets: inherit # pass all secrets
+    secrets: inherit # pass all secrets from initial workflow to nested
     
   forkless:
-    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }}
+    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }}  # Conditional check for draft & labels per job.
     uses: ./.github/workflows/forkless.yml
   
   node-only-update:
-    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }}
+    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'forkless')) }}  # Conditional check for draft & labels per job.
     uses: ./.github/workflows/node-only-update_v2.yml
     
   parallel_and_sequential_tests:
-    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'para')) }}
+    if: ${{ (github.event.pull_request.draft == false && contains( github.event.pull_request.labels.*.name, 'para')) }}      # Conditional check for draft & labels per job.
     uses: ./.github/workflows/node-only-update_v3.yml
 
   codestyle:
-    if: github.event.pull_request.draft == false
+    if: github.event.pull_request.draft == false                                                                             # Conditional check for draft per job.
     uses: ./.github/workflows/codestyle_v2.yml
   
   yarn_eslint:
-    if: github.event.pull_request.draft == false
+    if: github.event.pull_request.draft == false                                                                             # Conditional check for draft per job.
     uses: ./.github/workflows/test_codestyle_v2.yml
modified.github/workflows/ci-master.ymldiffbeforeafterboth
--- a/.github/workflows/ci-master.yml
+++ b/.github/workflows/ci-master.yml
@@ -1,14 +1,18 @@
+# Workflow which controls starts nested workflows.
 name: master
 
+# Triger: PR at 'master' branch with following types of events.
 on:
   pull_request:
     branches: [ 'master' ]
     types: [ opened, reopened, synchronize, ready_for_review ]
 
+#Concurency group for control execution queue over github runners.
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+# List of a jobs included into Workflow.
 jobs:
 
   unit-test:
@@ -22,11 +26,11 @@
 
   canary:
     uses: ./.github/workflows/canary.yml
-    secrets: inherit # pass all secrets
+    secrets: inherit # pass all secrets from initial workflow to nested
 
   xcm:
     uses: ./.github/workflows/xcm.yml
-    secrets: inherit # pass all secrets
+    secrets: inherit # pass all secrets from initial workflow to nested
 
   codestyle:
-    uses: ./.github/workflows/codestyle_v2.yml
\ No newline at end of file
+    uses: ./.github/workflows/codestyle_v2.yml
modified.github/workflows/codestyle_v2.ymldiffbeforeafterboth
--- a/.github/workflows/codestyle_v2.yml
+++ b/.github/workflows/codestyle_v2.yml
@@ -1,5 +1,8 @@
+# Nested workflow for checks related to formatting Rust code 
+
 name: cargo fmt
 
+# Triger: only call from main workflow(re-usable workflows)
 on:
   workflow_call:
 
modified.github/workflows/dev-build-tests_v2.ymldiffbeforeafterboth
--- a/.github/workflows/dev-build-tests_v2.yml
+++ b/.github/workflows/dev-build-tests_v2.yml
@@ -1,8 +1,9 @@
+# Integration test in --dev mode
+# https://cryptousetech.atlassian.net/browse/CI-12
 name: yarn test dev
 
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
 on:
- # Triggers the workflow on push or pull request events but only for the master branch
   workflow_call:
 
 
@@ -101,4 +102,4 @@
         run: |
           docker builder prune -f -a
           docker system prune -f
-          docker image prune -f -a
\ No newline at end of file
+          docker image prune -f -a
modified.github/workflows/forkless-update-data_v2.ymldiffbeforeafterboth
--- a/.github/workflows/forkless-update-data_v2.yml
+++ b/.github/workflows/forkless-update-data_v2.yml
@@ -1,11 +1,10 @@
-# Controls when the action will run.
+# Forkless update with data replication
+# https://cryptousetech.atlassian.net/browse/CI-42
+
+# Triger: only call from main workflow(re-usable workflows)
 on:
   workflow_call:
-
 
-#Define Workflow variables
-env:
-  REPO_URL: ${{ github.server_url }}/${{ github.repository }}
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
@@ -163,3 +162,10 @@
       - name: Stop running containers
         if: always()                   # run this step always
         run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down --volumes
+        
+      - name: Remove builder cache
+        if: always()                   # run this step always
+        run: |
+          docker builder prune -f -a
+          docker system prune -f
+          docker image prune -f -a
modified.github/workflows/forkless-update-nodata_v2.ymldiffbeforeafterboth
--- a/.github/workflows/forkless-update-nodata_v2.yml
+++ b/.github/workflows/forkless-update-nodata_v2.yml
@@ -1,6 +1,7 @@
+# Forkless update without data replication
+# https://cryptousetech.atlassian.net/browse/CI-41
 
-
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
 on:
   workflow_call:
 
@@ -213,3 +214,10 @@
       - name: Stop running containers
         if: always()                   # run this step always
         run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.${{ matrix.network }}.yml" down
+        
+      - name: Remove builder cache
+        if: always()                   # run this step always
+        run: |
+          docker builder prune -f -a
+          docker system prune -f
+          docker image prune -f -a
modified.github/workflows/forkless.ymldiffbeforeafterboth
--- a/.github/workflows/forkless.yml
+++ b/.github/workflows/forkless.yml
@@ -1,3 +1,5 @@
+# Intermediate Nested Workflow for calling subworkflows as a parallel tasks.
+
 name: Nesting Forkless
 
 on:
modified.github/workflows/market-test_v2.ymldiffbeforeafterboth
--- a/.github/workflows/market-test_v2.yml
+++ b/.github/workflows/market-test_v2.yml
@@ -1,3 +1,6 @@
+#https://cryptousetech.atlassian.net/browse/CI-37
+# Nested workflow for lunching Market e2e tests from external repository https://github.com/UniqueNetwork/market-e2e-tests
+
 name: market api tests
 
 # Controls when the action will run.
@@ -22,14 +25,8 @@
         include:
           - network: "opal"
             features: "opal-runtime"
-#          - network: "quartz"
-#            features: "quartz-runtime"
-#          - network: "unique"
-#            features: "unique-runtime"
 
     steps:
-
-
       - name: Clean Workspace
         uses: AutoModality/action-clean@v1.1.0
 
@@ -188,10 +185,3 @@
         run: |
           docker builder prune -f
           docker system prune -f
-
-      - name: Clean Workspace
-        if: always()
-        uses: AutoModality/action-clean@v1.1.0
-
-
-
modified.github/workflows/node-only-update_v2.ymldiffbeforeafterboth
--- a/.github/workflows/node-only-update_v2.yml
+++ b/.github/workflows/node-only-update_v2.yml
@@ -1,12 +1,13 @@
+# https://cryptousetech.atlassian.net/browse/CI-85
+# Node only update with restart polkadot-launch process.
+
 name: nodes-only update
 
-# Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
 on:
   workflow_call:
-#Define Workflow variables
-env:
-  REPO_URL: ${{ github.server_url }}/${{ github.repository }}
 
+
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
 
modified.github/workflows/node-only-update_v3.ymldiffbeforeafterboth
before Β· .github/workflows/node-only-update_v3.yml
1name: Parallele tests23# Controls when the action will run.4on:5  workflow_call:6#Define Workflow variables78# A workflow run is made up of one or more jobs that can run sequentially or in parallel9jobs:1011  nodes-execution-matrix:1213    name: execution matrix1415    runs-on: self-hosted-ci16    outputs:17      matrix: ${{ steps.create_matrix.outputs.matrix }}1819    steps:2021      - name: Clean Workspace22        uses: AutoModality/action-clean@v1.1.02324      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it25      - uses: actions/checkout@v326        with:27          ref: ${{ github.head_ref }}  #Checking out head commit2829      - name: Read .env file30        uses: xom9ikk/dotenv@v1.0.23132      - name: Create Execution matrix33        uses: fabiocaccamo/create-matrix-action@v234        id: create_matrix35        with:36          matrix: |37            network {opal}, runtime {opal}, features {opal-runtime}, mainnet_branch {${{ env.OPAL_MAINNET_TAG }}}38            network {quartz}, runtime {quartz}, features {quartz-runtime}, mainnet_branch {${{ env.QUARTZ_MAINNET_TAG }}}39            network {unique}, runtime {unique}, features {unique-runtime}, mainnet_branch {${{ env.UNIQUE_MAINNET_TAG }}}40 41  parallel-test:42    needs: nodes-execution-matrix43    # The type of runner that the job will run on44    runs-on: [self-hosted-ci,large]45464748    timeout-minutes: 13804950    name: ${{ matrix.network }} - parallel5152    continue-on-error: true         #Do not stop testing of matrix runs failed.  As it decided during PR review - it required 50/50& Let's check it with false.5354    strategy:55      matrix:56        include: ${{fromJson(needs.nodes-execution-matrix.outputs.matrix)}}5758    steps:5960      - name: Clean Workspace61        uses: AutoModality/action-clean@v1.1.06263      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it64      - uses: actions/checkout@v365        with:66          ref: ${{ github.head_ref }}  #Checking out head commit6768      - name: Read .env file69        uses: xom9ikk/dotenv@v1.0.27071      - name: Generate ENV related extend file for docker-compose72        uses: cuchi/jinja2-action@v1.2.073        with:74          template: .docker/docker-compose.tmp-node.j275          output_file: .docker/docker-compose.node.${{ matrix.network }}.yml76          variables: |77            REPO_URL=${{ github.server_url }}/${{ github.repository }}.git78            RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}79            POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }}80            POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }}81            MAINNET_TAG=${{ matrix.mainnet_tag }}82            MAINNET_BRANCH=${{ matrix.mainnet_branch }}83            FEATURE=${{ matrix.features }}84            RUNTIME=${{ matrix.runtime }}85            BRANCH=${{ github.head_ref }}86            87      - name: Show build configuration88        run: cat .docker/docker-compose.node.${{ matrix.network }}.yml8990      - name: Generate launch-config-forkless-nodata.json91        uses: cuchi/jinja2-action@v1.2.092        with:93          template: .docker/forkless-config/launch-config-node-update-only-v3.j294          output_file: .docker/launch-config-forkless-nodata.json95          variables: |96            FEATURE=${{ matrix.features }}97            RUNTIME=${{ matrix.runtime }}98            99      - name: Show launch-config-forkless configuration100        run: cat .docker/launch-config-forkless-nodata.json101102      - uses: actions/setup-node@v3103        with:104          node-version: 16105106      - name: Build the stack107        run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.node.${{ matrix.network }}.yml" up -d --build --remove-orphans --force-recreate --timeout 300108109      #  πŸš€ POLKADOT LAUNCH COMPLETE πŸš€110      - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade.111        if: success()112        run: |113          counter=160114          function check_container_status {115                docker inspect -f {{.State.Running}} node-parachain116          }117          function do_docker_logs {118                docker logs --details node-parachain  2>&1119          }120          function is_started {121                if [ "$(check_container_status)" == "true" ]; then122                        echo "Container: node-parachain RUNNING";123                        echo "Check Docker logs"124                        DOCKER_LOGS=$(do_docker_logs)125                        if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then126                                echo "πŸš€ POLKADOT LAUNCH COMPLETE πŸš€"127                                return 0128                        else129                                echo "Message not found in logs output, repeating..."130                                return 1131                        fi132                else133                        echo "Container node-parachain NOT RUNNING"134                        echo "Halting all future checks"135                        exit 1136                fi137          echo "something goes wrong"138          exit 1139          }140          while ! is_started; do141                echo "Waiting for special message in log files "142                sleep 30s143                counter=$(( $counter - 1 ))144                echo "Counter: $counter"145                if [ "$counter" -gt "0" ]; then146                         continue147                else148                         break149                fi150          done151          echo "Halting script"152          exit 0153        shell: bash154155      - name: Run Parallel tests on Node Parachain156        working-directory: tests157        run: |158          yarn install159          yarn add mochawesome160          echo "Ready to start tests"161          yarn polkadot-types162          NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW}163        env:164          RPC_URL: http://127.0.0.1:9933/165166      - name: Parallel Tests report167        uses: phoenix-actions/test-reporting@v8168        id: test-report-parallel169        if: success() || failure()    # run this step even if previous step failed170        with:171          name: Parallel Tests report - ${{ matrix.network }}            # Name of the check run which will be created172          path: tests/mochawesome-report/test-parallel-*.json    # Path to test results173          reporter: mochawesome-json174          fail-on-error: 'false'175    176             177      - name: Stop running containers178        if: always()                   # run this step always179        run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.node.${{ matrix.network }}.yml" down --volumes180181      - name: Remove builder cache182        if: always()                   # run this step always183        run: |184          docker builder prune -f -a185          docker system prune -f186          docker image prune -f -a187          188      - name: List files in Workspace189        if: always()190        run: ls -la ./191192  sequential-test:193    needs: nodes-execution-matrix194    # The type of runner that the job will run on195    runs-on: [self-hosted-ci,large]196197198199    timeout-minutes: 1380200201    name: ${{ matrix.network }} - sequential202203    continue-on-error: true         #Do not stop testing of matrix runs failed.  As it decided during PR review - it required 50/50& Let's check it with false.204205    strategy:206      matrix:207        include: ${{fromJson(needs.nodes-execution-matrix.outputs.matrix)}}208209    steps:210211      - name: Clean Workspace212        uses: AutoModality/action-clean@v1.1.0213214      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it215      - uses: actions/checkout@v3216        with:217          ref: ${{ github.head_ref }}  #Checking out head commit218219      - name: Read .env file220        uses: xom9ikk/dotenv@v1.0.2221222      - name: Generate ENV related extend file for docker-compose223        uses: cuchi/jinja2-action@v1.2.0224        with:225          template: .docker/docker-compose.tmp-node.j2226          output_file: .docker/docker-compose.node.${{ matrix.network }}.yml227          variables: |228            REPO_URL=${{ github.server_url }}/${{ github.repository }}.git229            RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }}230            POLKADOT_BUILD_BRANCH=${{ env.POLKADOT_BUILD_BRANCH }}231            POLKADOT_MAINNET_BRANCH=${{ env.POLKADOT_MAINNET_BRANCH }}232            MAINNET_TAG=${{ matrix.mainnet_tag }}233            MAINNET_BRANCH=${{ matrix.mainnet_branch }}234            FEATURE=${{ matrix.features }}235            RUNTIME=${{ matrix.runtime }}236            BRANCH=${{ github.head_ref }}237            238      - name: Show build configuration239        run: cat .docker/docker-compose.node.${{ matrix.network }}.yml240241      - name: Generate launch-config-forkless-nodata.json242        uses: cuchi/jinja2-action@v1.2.0243        with:244          template: .docker/forkless-config/launch-config-node-update-only-v3.j2245          output_file: .docker/launch-config-forkless-nodata.json246          variables: |247            FEATURE=${{ matrix.features }}248            RUNTIME=${{ matrix.runtime }}249            250      - name: Show launch-config-forkless configuration251        run: cat .docker/launch-config-forkless-nodata.json252253      - uses: actions/setup-node@v3254        with:255          node-version: 16256257      - name: Build the stack258        run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.node.${{ matrix.network }}.yml" up -d --build --remove-orphans --force-recreate --timeout 300259260      #  πŸš€ POLKADOT LAUNCH COMPLETE πŸš€261      - name: Check if docker logs consist messages related to testing of Node Parachain Upgrade.262        if: success()263        run: |264          counter=160265          function check_container_status {266                docker inspect -f {{.State.Running}} node-parachain267          }268          function do_docker_logs {269                docker logs --details node-parachain  2>&1270          }271          function is_started {272                if [ "$(check_container_status)" == "true" ]; then273                        echo "Container: node-parachain RUNNING";274                        echo "Check Docker logs"275                        DOCKER_LOGS=$(do_docker_logs)276                        if [[ ${DOCKER_LOGS} = *"POLKADOT LAUNCH COMPLETE"* ]];then277                                echo "πŸš€ POLKADOT LAUNCH COMPLETE πŸš€"278                                return 0279                        else280                                echo "Message not found in logs output, repeating..."281                                return 1282                        fi283                else284                        echo "Container node-parachain NOT RUNNING"285                        echo "Halting all future checks"286                        exit 1287                fi288          echo "something goes wrong"289          exit 1290          }291          while ! is_started; do292                echo "Waiting for special message in log files "293                sleep 30s294                counter=$(( $counter - 1 ))295                echo "Counter: $counter"296                if [ "$counter" -gt "0" ]; then297                         continue298                else299                         break300                fi301          done302          echo "Halting script"303          exit 0304        shell: bash305306      - name: Run Sequential tests on Node Parachain307        working-directory: tests308        run: |309          yarn install310          yarn add mochawesome311          echo "Ready to start tests"312          yarn polkadot-types313          NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW}314        env:315          RPC_URL: http://127.0.0.1:9933/316317      - name: Sequential Tests report318        uses: phoenix-actions/test-reporting@v8319        id: test-report-sequential320        if: success() || failure()    # run this step even if previous step failed321        with:322          name: Parallel Tests report - ${{ matrix.network }}            # Name of the check run which will be created323          path: tests/mochawesome-report/test-sequential-*.json    # Path to test results324          reporter: mochawesome-json325          fail-on-error: 'false'326       327      - name: Stop running containers328        if: always()                   # run this step always329        run: docker-compose -f ".docker/docker-compose-forkless.yml" -f ".docker/docker-compose.node.${{ matrix.network }}.yml" down --volumes330331      - name: Remove builder cache332        if: always()                   # run this step always333        run: |334          docker builder prune -f -a335          docker system prune -f336          docker image prune -f -a337          338      - name: List files in Workspace339        if: always()340        run: ls -la ./
modified.github/workflows/test_codestyle_v2.ymldiffbeforeafterboth
--- a/.github/workflows/test_codestyle_v2.yml
+++ b/.github/workflows/test_codestyle_v2.yml
@@ -1,5 +1,8 @@
+# Yarn Eslint over tests
+#
 name: yarn eslint
 
+# Triger: only call from main workflow(re-usable workflows)
 on:
   workflow_call:
 
modified.github/workflows/try-runtime_v2.ymldiffbeforeafterboth
--- a/.github/workflows/try-runtime_v2.yml
+++ b/.github/workflows/try-runtime_v2.yml
@@ -1,3 +1,7 @@
+# Try-runtime checks
+# https://cryptousetech.atlassian.net/browse/CI-38
+
+# Triger: only call from main workflow(re-usable workflows)
 on:
   workflow_call:
 
@@ -66,3 +70,10 @@
       - name: Stop running containers
         if: always()                   # run this step always
         run: docker-compose -f ".docker/docker-compose-try-runtime.yml" -f ".docker/docker-compose.try-runtime.${{ matrix.network }}.yml" down
+        
+      - name: Remove builder cache
+        if: always()                   # run this step always
+        run: |
+          docker builder prune -f -a
+          docker system prune -f
+          docker image prune -f -a
modified.github/workflows/unit-test_v2.ymldiffbeforeafterboth
--- a/.github/workflows/unit-test_v2.yml
+++ b/.github/workflows/unit-test_v2.yml
@@ -1,8 +1,9 @@
+# Re-Usable Workflow for lanching Unit tests
 name: unit tests
 
 # Controls when the action will run.
+# Triger: only call from main workflow(re-usable workflows)
 on:
- # Triggers the workflow on push or pull request events but only for the master branch
   workflow_call:
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
@@ -49,3 +50,10 @@
       - name: Stop running containers
         if: always()                   # run this step always
         run: docker-compose -f ".docker/docker-compose-dev.yaml" -f ".docker/docker-compose.unit.yml" down
+        
+      - name: Remove builder cache
+        if: always()                   # run this step always
+        run: |
+          docker builder prune -f -a
+          docker system prune -f
+          docker image prune -f -a