--- /dev/null +++ b/.baedeker/forkless-data.jsonnet @@ -0,0 +1,67 @@ +local +m = import 'baedeker-library/mixin/spec.libsonnet', +rm = import 'baedeker-library/mixin/raw-spec.libsonnet', +; + +function(relay_spec, forked_spec, fork_source) + +local relay = { + name: 'relay', + bin: 'bin/polkadot', + validatorIdAssignment: 'staking', + spec: {Genesis:{ + chain: relay_spec, + modify:: m.genericRelay($), + }}, + nodes: { + [name]: { + bin: $.bin, + wantedKeys: 'relay', + }, + for name in ['alice', 'bob', 'charlie', 'dave', 'eve'] + }, +}; + +local unique = { + name: 'unique', + bin: 'bin/unique', + paraId: 1001, + spec: {Raw:{ + local modifyRaw = bdk.mixer([ + rm.resetNetworking($), + rm.decodeSpec(), + rm.polkaLaunchPara($), + rm.reencodeSpec(), + ]), + raw_spec: modifyRaw({ + name: "Unused", + id: "%s_local" % forked_spec, + bootNodes: error "override me", + chainType: error "override me", + telemetryEndpoints: error "override me", + codeSubstitutes: error "override me", + para_id: error "override me", + relay_chain: "unused", + genesis: { + raw: { + top: cql.chain(fork_source).latest._preloadKeys._raw, + childrenDefault: {}, + }, + }, + }), + }}, + nodes: { + [name]: { + bin: $.bin, + wantedKeys: 'para', + }, + for name in ['alice', 'bob'] + }, +}; + +relay + { + parachains: { + [para.name]: para, + for para in [unique] + }, +} --- a/.github/workflows/forkless-update-data.yml +++ b/.github/workflows/forkless-update-data.yml @@ -9,9 +9,9 @@ # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - prepare-execution-marix: + prepare-execution-matrix: - name: Prepare execution matrix + name: execution matrix runs-on: self-hosted-ci outputs: @@ -34,27 +34,30 @@ uses: CertainLach/create-matrix-action@v4 id: create_matrix with: - # TODO: Remove state-version-0 after mainnet upgrade matrix: | - network {quartz}, wasm_name {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, replica_from_address {${{ env.QUARTZ_REPLICA_FROM }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}, extra_features {,state-version-0} - network {unique}, wasm_name {unique}, mainnet_branch {${{ env.UNIQUE_MAINNET_BRANCH }}}, replica_from_address {${{ env.UNIQUE_REPLICA_FROM }}}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}, extra_features {,state-version-0} - network {opal}, wasm_name {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}, replica_from_address {${{ env.OPAL_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}, extra_features {,state-version-0} - network {sapphire}, wasm_name {quartz}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, replica_from_address {${{ env.SAPPHIRE_REPLICA_FROM }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}, extra_features {,state-version-0} + network {opal}, mainnet_branch {${{ env.OPAL_MAINNET_BRANCH }}}, relay_branch {${{ env.UNIQUEWEST_MAINNET_BRANCH }}}, runtime_features {opal-runtime}, fork_source {${{ env.OPAL_REPLICA_FROM }}} + network {sapphire}, mainnet_branch {${{ env.SAPPHIRE_MAINNET_BRANCH }}}, relay_branch {${{ env.UNIQUEEAST_MAINNET_BRANCH }}}, runtime_features {sapphire-runtime}, wasm_name {quartz}, fork_source {${{ env.SAPPHIRE_REPLICA_FROM }}} + network {quartz}, mainnet_branch {${{ env.QUARTZ_MAINNET_BRANCH }}}, relay_branch {${{ env.KUSAMA_MAINNET_BRANCH }}}, runtime_features {quartz-runtime}, wasm_name {quartz}, fork_source {${{ env.QUARTZ_REPLICA_FROM }}} + network {unique}, mainnet_branch {${{ env.UNIQUE_MAINNET_BRANCH }}}, relay_branch {${{ env.POLKADOT_MAINNET_BRANCH }}}, runtime_features {unique-runtime}, wasm_name {unique}, fork_source {${{ env.UNIQUE_REPLICA_FROM }}} forkless-data-build: - needs: prepare-execution-marix + + needs: prepare-execution-matrix # The type of runner that the job will run on runs-on: [self-hosted-ci,large] + timeout-minutes: 1380 name: ${{ matrix.network }}-data-build + + continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. + strategy: matrix: - include: ${{fromJson(needs.prepare-execution-marix.outputs.matrix)}} - - continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. + include: ${{fromJson(needs.prepare-execution-matrix.outputs.matrix)}} steps: + - name: Clean Workspace uses: AutoModality/action-clean@v1.1.0 @@ -62,203 +65,128 @@ - uses: actions/checkout@v3.1.0 with: ref: ${{ github.head_ref }} #Checking out head commit - - - name: Read .env file - uses: xom9ikk/dotenv@v2 # Prepare SHA - name: Prepare SHA uses: ./.github/actions/prepare - # Build main image for FORKLESS-UPDATE-NODATA - - name: Generate ENV related extend Dockerfile file + - name: Read .env file + uses: xom9ikk/dotenv@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2.1.0 + with: + username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} + password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} + + - name: Generate ENV related extend Dockerfile file for POLKADOT uses: cuchi/jinja2-action@v1.2.0 with: - template: .docker/Dockerfile-parachain-upgrade-data.j2 - output_file: .docker/Dockerfile-parachain-upgrade-data.${{ matrix.network }}.yml + template: .docker/Dockerfile-polkadot.j2 + output_file: .docker/Dockerfile-polkadot.${{ matrix.relay_branch }}.yml variables: | RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} - NETWORK=${{ matrix.network }} - MAINNET_BRANCH=${{ matrix.mainnet_branch }} - WASM_NAME=${{ matrix.wasm_name }} - RELAY_CHAIN_TYPE=${{ env.RELAY_CHAIN_TYPE }} - DESTINATION_SPEC_VERSION=${{ env.DESTINATION_SPEC_VERSION }} POLKADOT_BUILD_BRANCH=${{ matrix.relay_branch }} - REPLICA_FROM=${{ matrix.replica_from_address }} - CHAINQL=${{ env.CHAINQL }} - EXTRA_FEATURES=${{ matrix.extra_features }} - - - name: Show build configuration - run: cat .docker/Dockerfile-parachain-upgrade-data.${{ matrix.network }}.yml - - - name: Run find-and-replace to remove slashes from branch name - uses: mad9000/actions-find-and-replace-string@4 - id: branchname + + - name: Prepare polkadot + uses: ./.github/actions/buildContainer + id: polkadot with: - source: ${{ github.head_ref }} - find: '/' - replace: '-' + container: uniquenetwork/builder-polkadot + tag: ${{ matrix.relay_branch }} + context: .docker + dockerfile: Dockerfile-polkadot.${{ matrix.relay_branch }}.yml + dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} + dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - name: Set build SHA - shell: bash - run: | - echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV - - - name: Build the stack - run: cd .docker/ && docker build --no-cache --file ./Dockerfile-parachain-upgrade-data.${{ matrix.network }}.yml --tag uniquenetwork/ci-forkless-data-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA ../ - - - name: Log in to Docker Hub - uses: docker/login-action@v2.1.0 + - name: Prepare mainnet + uses: ./.github/actions/buildContainer + id: mainnet with: - username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - - name: Push docker image version - run: docker push uniquenetwork/ci-forkless-data-local:${{ matrix.network }}-${{ steps.branchname.outputs.value }}-$BUILD_SHA - - - name: Remove builder cache - if: always() # run this step always - run: | - docker builder prune -f - docker system prune -f - - forkless-data-tests: - needs: [prepare-execution-marix, forkless-data-build] - # The type of runner that the job will run on - runs-on: [self-hosted-ci, large] - - timeout-minutes: 600 - - name: ${{ matrix.network }}-data-tests - - continue-on-error: true #Do not stop testing of matrix runs failed. As it decided during PR review - it required 50/50& Let's check it with false. - - strategy: - matrix: - include: ${{fromJson(needs.prepare-execution-marix.outputs.matrix)}} - - steps: - - name: Skip if pull request is in Draft - if: github.event.pull_request.draft == true - run: exit 1 - - - name: Clean Workspace - uses: AutoModality/action-clean@v1.1.0 + container: uniquenetwork/ci-data-${{ matrix.network }} + tag: ${{ matrix.mainnet_branch }} + context: .docker + dockerfile: Dockerfile-unique-release + args: | + --build-arg RUNTIME_FEATURES=${{ matrix.runtime_features }} + --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} + --build-arg UNIQUE_VERSION=${{ matrix.mainnet_branch }} + dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} + dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v3.1.0 + - name: Prepare latest + uses: ./.github/actions/buildContainer + id: latest with: - ref: ${{ github.head_ref }} #Checking out head commit - - - name: Prepare - uses: ./.github/actions/prepare + container: uniquenetwork/ci-data-${{ matrix.network }} + tag: ${{ env.REF_SLUG }}-${{ env.BUILD_SHA }} + context: . + dockerfile: .docker/Dockerfile-unique + args: | + --build-arg RUNTIME_FEATURES=${{ matrix.runtime_features }} + --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} + dockerhub_username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} + dockerhub_token: ${{ secrets.CORE_DOCKERHUB_TOKEN }} - - name: Set build SHA - shell: bash - run: | - echo "BUILD_SHA=${LAST_COMMIT_SHA:0:8}" >> $GITHUB_ENV - - - name: Run find-and-replace to remove slashes from branch name - uses: mad9000/actions-find-and-replace-string@4 - id: branchname + - name: Extract wasms + uses: ./.github/actions/extractDocker + id: wasms with: - source: ${{ github.head_ref }} - find: '/' - replace: '-' + image: ${{ steps.latest.outputs.name }} + directory: /wasm - - name: Read .env file - uses: xom9ikk/dotenv@v2 - - - name: Generate ENV related extend file for docker-compose - uses: cuchi/jinja2-action@v1.2.0 + - uses: actions/setup-node@v3.5.1 with: - template: .docker/docker-compose.forkless-data.j2 - output_file: .docker/docker-compose.forkless-data.${{ matrix.network }}.yml - variables: | - NETWORK=${{ matrix.network }} - BUILD_TAG=${{ steps.branchname.outputs.value }}-$BUILD_SHA - - - name: Show build configuration - run: cat .docker/docker-compose.forkless-data.${{ matrix.network }}.yml + node-version: 18 - - name: Log in to Docker Hub - uses: docker/login-action@v2.1.0 - with: - username: ${{ secrets.CORE_DOCKERHUB_USERNAME }} - password: ${{ secrets.CORE_DOCKERHUB_TOKEN }} + - name: Install baedeker + uses: UniqueNetwork/baedeker-action/setup@built - - name: Build the stack - run: docker-compose -f ".docker/docker-compose.forkless-data.${{ matrix.network }}.yml" up -d --remove-orphans --force-recreate --timeout 300 + - name: Setup library + run: mkdir -p .baedeker/vendor/ && git clone https://github.com/UniqueNetwork/baedeker-library .baedeker/vendor/baedeker-library - - uses: actions/setup-node@v3.5.1 + - name: Start network + uses: UniqueNetwork/baedeker-action@built + id: bdk with: - node-version: 18 + jpath: | + .baedeker/vendor + tla-str: | + relay_spec=rococo-local + forked_spec=${{ matrix.network }} + fork_source=${{ matrix.fork_source }} + inputs: | + .baedeker/forkless-data.jsonnet + snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/polkadot':{dockerImage:'${{ steps.polkadot.outputs.name }}'}}) + snippet:(import 'baedeker-library/ops/rewrites.libsonnet').rewriteNodePaths({'bin/unique':{dockerImage:'${{ steps.mainnet.outputs.name }}'}}, extra_node_mixin = {legacyRpc: true}) - - name: Check if docker logs consist logs related to Runtime Upgrade testing. - if: success() + - name: "Reconcile: runtime is upgraded" + working-directory: tests run: | - counter=160 - function check_container_status { - docker inspect -f {{.State.Running}} forkless-data-${{ matrix.network }} - } - function do_docker_logs { - docker logs --details forkless-data-${{ matrix.network }} 2>&1 - } - function is_started { - if [ "$(check_container_status)" == "true" ]; then - echo "Container: forkless-data-${{ matrix.network }} RUNNING"; - echo "Check Docker logs" - DOCKER_LOGS=$(do_docker_logs) - if [[ ${DOCKER_LOGS} = *"🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸"* ]];then - echo "🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸" - return 0 - elif [[ ${DOCKER_LOGS} = *"🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧"* ]];then - echo "🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧" - return 1 - else - echo "Message not found in logs output, repeating..." - return 1 - fi - else - echo "Container forkless-data-${{ matrix.network }} not RUNNING" - echo "Halting all future checks" - exit 1 - fi - exit 0 - } - while ! is_started; do - echo "Waiting for special message in log files " - sleep 30s - counter=$(( $counter - 1 )) - echo "Counter: $counter" - if [ "$counter" -gt "0" ]; then - continue - else - echo "Counter reached zero, yet upgrade is not finished" - exit 1 - fi - done - echo "Halting script" - exit 0 - shell: bash + yarn + ./scripts/wait_for_first_block.sh + echo "Executing upgrade" + yarn ts-node --esm src/util/authorizeEnactUpgrade.ts ${{ steps.wasms.outputs.dir }}/${{ matrix.wasm_name }}-runtime/${{ matrix.wasm_name }}_runtime.compact.compressed.wasm + env: + RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - name: Collect Docker Logs - if: success() || failure() - uses: jwalton/gh-docker-logs@v2.2.1 - with: - dest: './forkless-parachain-upgrade-data-logs.${{ matrix.network }}' + - name: Run Parallel tests after forkless upgrade + working-directory: tests + run: | + ./scripts/wait_for_first_block.sh + echo "Ready to start tests" + NOW=$(date +%s) && yarn testParallel --reporter mochawesome --reporter-options reportFilename=test-parallel-${NOW} + env: + RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - - name: Show Docker logs + - name: Run Sequential tests after forkless upgrade if: success() || failure() - run: cat './forkless-parachain-upgrade-data-logs.${{ matrix.network }}/forkless-data-${{ matrix.network }}.log' - - - name: Stop running containers - if: always() # run this step always - run: docker-compose -f ".docker/docker-compose.forkless-data.${{ matrix.network }}.yml" down --volumes + working-directory: ${{ matrix.mainnet_branch }}/tests + run: NOW=$(date +%s) && yarn testSequential --reporter mochawesome --reporter-options reportFilename=test-sequential-${NOW} + env: + RPC_URL: ${{ env.RELAY_UNIQUE_HTTP_URL }} - name: Remove builder cache if: always() # run this step always run: | - docker builder prune -f -a docker system prune -f - docker image prune -f -a