difftreelog
refactor(zombienet) restore check for relay chain upgrade completion before additional waiting
in: master
1 file changed
tests/src/util/frankenstein.tsdiffbeforeafterboth1// Copyright 2019-2023 Unique Network (Gibraltar) Ltd.2// This file is part of Unique Network.34// Unique Network is free software: you can redistribute it and/or modify5// it under the terms of the GNU General Public License as published by6// the Free Software Foundation, either version 3 of the License, or7// (at your option) any later version.89// Unique Network is distributed in the hope that it will be useful,10// but WITHOUT ANY WARRANTY; without even the implied warranty of11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the12// GNU General Public License for more details.1314// You should have received a copy of the GNU General Public License15// along with Unique Network. If not, see <http://www.gnu.org/licenses/>.1617import {ApiPromise} from '@polkadot/api';18import {blake2AsHex, cryptoWaitReady} from '@polkadot/util-crypto';19import zombie from '@zombienet/orchestrator/dist';20import {readNetworkConfig} from '@zombienet/utils/dist';21import {resolve} from 'path';22import {usingPlaygrounds} from '.';23import fs from 'fs';2425const ZOMBIENET_CREDENTIALS = process.env.ZOMBIENET_CREDENTIALS || '../.env';26const NETWORK_CONFIG_FILE = process.argv[2] ?? '../launch-zombienet.toml';27const PARA_DIR = process.env.PARA_DIR || '../';28const RELAY_DIR = process.env.RELAY_DIR || '../../polkadot/';29const REPLICA_FROM = process.env.REPLICA_FROM || 'wss://ws-opal.unique.network:443';30const NEW_RELAY_BIN = process.env.NEW_RELAY_BIN;31const NEW_RELAY_WASM = process.env.NEW_RELAY_WASM;32const NEW_PARA_BIN = process.env.NEW_PARA_BIN;33const NEW_PARA_WASM = process.env.NEW_PARA_WASM;34const PARACHAIN_BLOCK_TIME = 12_000;3536let network: zombie.Network | undefined;3738// Stop the network if it is running39const stop = async () => {40 await network?.stop();41};4243// Promise of a timeout44function delay(ms: number) {45 return new Promise(resolve => setTimeout(resolve, ms));46}4748// Countdown with time left on display49async function waitWithTimer(time: number) {50 const secondsTotal = Math.ceil(time / 1000);51 for (let i = secondsTotal; i > 0; i--) {52 // could also introduce hours, but wth53 const seconds = i % 60;54 const text = `Time left: ${Math.floor(i/60)}:${seconds<10'0'+secondsseconds}`;55 if (process.stdout.isTTY)56 process.stdout.write(text);57 else if (seconds % 10 == 0)58 console.log(text);59 await delay(1000);60 if (process.stdout.isTTY) {61 process.stdout.clearLine(0);62 process.stdout.cursorTo(0);63 }64 }65}6667// Get the runtime's current version68function getSpecVersion(api: ApiPromise): number {69 return (api.consts.system.version as any).specVersion.toNumber();70}7172// Get the required information on the relay chain73function getRelayInfo(api: ApiPromise): {specVersion: number, epochBlockLength: number, blockTime: number, epochTime: number} {74 const info = {75 specVersion: getSpecVersion(api),76 epochBlockLength: (api.consts.babe.epochDuration as any).toNumber(),77 blockTime: (api.consts.babe.expectedBlockTime as any).toNumber(),78 epochTime: 0,79 };80 info.epochTime = info.epochBlockLength * info.blockTime;81 return info;82}8384// Enable or disable maintenance mode if present on the chain85async function toggleMaintenanceMode(value: boolean, wsUri: string) {86 await usingPlaygrounds(async (helper, privateKey) => {87 const superuser = await privateKey('//Alice');88 try {89 const toggle = value ? 'enable' : 'disable';90 await helper.getSudo().executeExtrinsic(superuser, `api.tx.maintenance.${toggle}`, []);91 console.log(`Maintenance mode ${value'engaged''disengaged'}.`);92 } catch (e) {93 console.error('Couldn\'t set maintenance mode. The maintenance pallet probably does not exist. Log:', e);94 }95 }, wsUri);96}9798const raiseZombienet = async (): Promise<void> => {99 const isUpgradeTesting = !!NEW_RELAY_BIN || !!NEW_RELAY_WASM || !!NEW_PARA_BIN || !!NEW_PARA_WASM;100 /*101 // If there is nothing to upgrade, what is the point102 if (!isUpgradeTesting) {103 console.warn('\nNeither the relay nor the parachain were selected for an upgrade! ' +104 'Please pass environment vars `NEW_RELAY_BIN`, `NEW_RELAY_WASM`, `NEW_PARA_BIN`, `NEW_PARA_WASM`.');105 process.exit(1);106 }107 */108109 // an unsavory practice, but very convenient, mwahahah110 process.env.PARA_DIR = PARA_DIR;111 process.env.RELAY_DIR = RELAY_DIR;112 process.env.REPLICA_FROM = REPLICA_FROM;113114 const configPath = resolve(NETWORK_CONFIG_FILE);115 const networkConfig = readNetworkConfig(configPath);116 // console.log(networkConfig);117 if (networkConfig.settings.provider !== 'native') {118 throw new Error(`Oh no! Expected native network, got ${networkConfig.settings.provider}.`);119 }120121 await cryptoWaitReady();122123 // Launch Zombienet!124 network = await zombie.start(ZOMBIENET_CREDENTIALS, networkConfig, {silent: false});125126 // Get the relay chain info like the epoch length and spec version127 // Then restart each parachain's binaries128 // // Stop and restart each node129 // // Send specified keys to parachain nodes in case the parachain requires it130 // If it is not needed to upgrade runtimes themselves, the job is done!131132 // Record some required information regarding the relay chain133 await network.relay[0].connectApi();134 let relayInfo = getRelayInfo(network.relay[0].apiInstance!);135 await network.relay[0].apiInstance!.disconnect();136 if (isUpgradeTesting) {137 console.log('Relay stats:', relayInfo);138 }139140 // non-exported functionality of NativeClient141 const networkClient = (network.client as any);142143 if (NEW_RELAY_BIN) {144 console.log('\n🧶 Restarting relay nodes');145146 for (const [index, node] of network.relay.entries()) {147 await node.apiInstance?.disconnect();148149 console.log(`\n🚦 Starting timeout for the epoch change (node ${index+1}/${network.relay.length})...`);150 await waitWithTimer(relayInfo.epochTime);151152 // Replace the node-starting command with the new binary153 const cmd = networkClient.processMap[node.name].cmd[0].split(' ')[0];154 networkClient.processMap[node.name].cmd = networkClient.processMap[node.name].cmd.map((arg: string) => arg.replace(cmd, NEW_RELAY_BIN));155156 await node.restart();157 }158159 console.log('\n🌒 All relay nodes restarted with the new binaries.');160 }161162 if (NEW_PARA_BIN) {163 for (const paraId in network.paras) {164 const para = network.paras[paraId];165 console.log(`\n🧶 Restarting collator nodes of parachain ${paraId}`);166167 for (const [_index, node] of para.nodes.entries()) {168 await node.apiInstance?.disconnect();169170 // Replace the node-starting command with the new binary171 const cmd = networkClient.processMap[node.name].cmd[0].split(' ')[0];172 networkClient.processMap[node.name].cmd = networkClient.processMap[node.name].cmd.map((arg: string) => arg.replace(cmd, NEW_PARA_BIN));173174 await node.restart();175 // applyaurakey?176 // Zombienet handles it on first-time node creation177 }178 }179180 console.log('\n🌗 All parachain collators restarted with the new binaries.');181 }182183 // Re-establish connection to the relay node and get the runtime upgrade validation delay for parachains184 // For the relay, connect and set the new runtime code185 // For each parachain, connect, authorize and upgrade its runtime186 // Ping the the chains for the runtime upgrade after the minimal time and then every few blocks187 // // For each parachain, re-connect and verify that the runtime upgrade is successful188189 let relayUpgradeCompleted = false, paraUpgradeCompleted = false;190191 if (NEW_RELAY_WASM) {192 const relayOldVersion = relayInfo.specVersion;193 console.log('\n🚦 Starting timeout for the next epoch before upgrading the relay runtime code...');194 await waitWithTimer(relayInfo.epochTime);195196 console.log('--- Upgrading the relay chain runtime \t---');197198 // Read the new WASM code and set it as the relay's new code199 const code = fs.readFileSync(NEW_RELAY_WASM).toString('hex');200 await usingPlaygrounds(async (helper, privateKey) => {201 const superuser = await privateKey('//Alice');202203 const result = await helper.executeExtrinsic(204 superuser,205 'api.tx.sudo.sudoUncheckedWeight',206 [helper.constructApiCall('api.tx.system.setCode', [`0x${code}`]), 0],207 );208209 if (result.status == 'Fail') {210 console.error('Failed to upgrade the runtime:', result);211 }212213 // Get the updated information from the relay's new runtime214 relayInfo = getRelayInfo(helper.getApi());215 }, network.relay[0].wsUri);216217 if (relayOldVersion != relayInfo.specVersion) {218 // eslint-disable-next-line no-useless-escape219 console.log(`\n\🛰️ The relay has successfully upgraded from version ${relayOldVersion} to ${relayInfo.specVersion}!`);220 relayUpgradeCompleted = true;221 } else {222 console.error(`\nThe relay did not upgrade from version ${relayOldVersion}!`);223 }224 } else {225 // If the relay did not need to be upgraded, it's already technically completed226 relayUpgradeCompleted = true;227 }228229 if (NEW_PARA_WASM) {230 let codeValidationDelayBlocks = 0;231 const upgradingParas: {[id: string]: {version: number, upgraded: boolean}} = {};232 // Calculate the code validation delay of the relay chain,233 // so that we know how much to wait before the parachains can be upgraded after the extrinsic234 await usingPlaygrounds(async (helper) => {235 const {validationUpgradeDelay, minimumValidationUpgradeDelay} =236 (await helper.callRpc('api.query.configuration.activeConfig', [])).toJSON() as any;237238 codeValidationDelayBlocks = Math.max(validationUpgradeDelay ?? 0, minimumValidationUpgradeDelay ?? 0);239 }, network.relay[0].wsUri);240241 // Wait for the next epoch so that the parachains will start cooperating with the relay242 //if (relayUpgradeCompleted) {243 console.log('\n🚥 Starting timeout for the next epoch before upgrading the parachains code...');244 await waitWithTimer(relayInfo.epochTime);245 //}246247 for (const paraId in network.paras) {248 console.log(`\n--- Upgrading the runtime of parachain ${paraId} \t---`);249 const para = network.paras[paraId];250251 // Enable maintenance mode if present252 await toggleMaintenanceMode(true, para.nodes[0].wsUri);253254 // Read the WASM code and authorize the upgrade with its hash and set it as the new runtime255 const code = fs.readFileSync(NEW_PARA_WASM);256 const codeHash = blake2AsHex(code);257 await usingPlaygrounds(async (helper, privateKey) => {258 const superuser = await privateKey('//Alice');259260 upgradingParas[paraId] = {version: getSpecVersion(helper.getApi()), upgraded: false};261262 console.log('--- Authorizing the parachain runtime upgrade \t---');263 let result = await helper.executeExtrinsic(264 superuser,265 'api.tx.sudo.sudoUncheckedWeight',266 [helper.constructApiCall('api.tx.parachainSystem.authorizeUpgrade', [codeHash]), 0],267 );268269 if (result.status == 'Fail') {270 console.error('Failed to authorize the upgrade:', result);271 return;272 }273274 console.log('--- Enacting the parachain runtime upgrade \t---');275 result = await helper.executeExtrinsic(276 superuser,277 'api.tx.sudo.sudoUncheckedWeight',278 [helper.constructApiCall('api.tx.parachainSystem.enactAuthorizedUpgrade', [`0x${code.toString('hex')}`]), 0],279 );280281 if (result.status == 'Fail') {282 console.error('Failed to upgrade the runtime:', result);283 }284 }, para.nodes[0].wsUri);285 }286287 // Check the upgrades of the parachains, first after the minimum code validation delay, and then after some block time increments288 let firstPass = true;289 for (let attempt = 0; attempt < 3 && !paraUpgradeCompleted; attempt++) {290 if (firstPass) {291 console.log('\nCode validation delay:', codeValidationDelayBlocks, 'blocks');292 console.log('🚥 Waiting for the minimum code validation delay before the parachain can upgrade...');293 await waitWithTimer(relayInfo.blockTime * codeValidationDelayBlocks);294 firstPass = false;295 } else {296 console.log('\n🚥 Waiting for a few blocks more to verify that the parachain upgrades are successful...');297 await waitWithTimer(PARACHAIN_BLOCK_TIME * 3);298 }299300 // Ping the parachains' nodes for new runtime versions301 let upgradeFailed = false;302 for (const paraId in network.paras) {303 if (upgradingParas[paraId].upgraded) continue;304305 const para = network.paras[paraId];306 // eslint-disable-next-line require-await307 await usingPlaygrounds(async (helper) => {308 const specVersion = getSpecVersion(helper.getApi());309310 if (specVersion != upgradingParas[paraId].version) {311 // eslint-disable-next-line no-useless-escape312 console.log(`\n\🛰️ Parachain ${paraId} has successfully upgraded from version ${upgradingParas[paraId].version} to ${specVersion}!`);313 upgradingParas[paraId].upgraded = true;314 } else {315 console.error(`\nParachain ${paraId} failed to upgrade from version ${upgradingParas[paraId].version}!`);316 upgradeFailed = true;317 }318 }, para.nodes[0].wsUri);319320 paraUpgradeCompleted = !upgradeFailed;321 }322 }323324 // Disable maintenance mode if present325 for (const paraId in network.paras) {326 await toggleMaintenanceMode(false, network.paras[paraId].nodes[0].wsUri);327 }328 } else {329 // If the relay did not need to be upgraded, it's already technically completed330 paraUpgradeCompleted = true;331 }332333 // await network.stop();334335 if (isUpgradeTesting) {336 if (paraUpgradeCompleted && relayUpgradeCompleted) {337 console.log("\n🛸 PARACHAINS' RUNTIME UPGRADE TESTING COMPLETE 🛸");338 } else {339 console.error("\n🚧 PARACHAINS' RUNTIME UPGRADE TESTING FAILED 🚧");340 }341 } else {342 console.log('🚀 ZOMBIENET RAISED 🚀');343 }344};345346raiseZombienet()/*.then(async () => await stop())*/.catch(async (e) => {347 console.error(e);348 await stop();349 process.exit(1);350});