git.delta.rocks / unique-network / refs/commits / 6c9848e80380

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/README.md
@@ -0,0 +1,42 @@
+# Update Procedure
+
+- Enable maintenance mode
+- [Collect migration data using ChainQL](#stakers-data-loading)
+- ❗️❗️❗️ Initiate the runtime upgrade only at this point ❗️❗️❗️
+- Wait for the upgrade to complete
+- [Execute offchain migration](#execute-offchain-migration)
+- Disable maintenance mode
+
+## Stakers Data Loading
+
+Set the environment variable (WS_RPC). For example, ws://localhost:9944. Execute the following command:
+
+```sh
+chainql --tla-str=chainUrl=<WS_RPC> stakersParser.jsonnet > output.json
+```
+
+where `<WS_RPC>` - is the network address.
+
+Example for Opal:
+
+```sh
+chainql --tla-str=chainUrl=wss://eu-ws-opal.unique.network:443 stakersParser.jsonnet > output.json
+```
+
+To install chainql, execute the following command:
+
+```sh
+cargo install chainql
+```
+
+## Execute offchain migration
+
+To run, you need to set an environment variables:
+- `SUPERUSER_SEED` – the sudo key seed.
+- `WS_RPC` – the network address
+
+Run the migration by executing the following command:
+
+```sh
+npx ts-node --esm executeMigration.ts
+```

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/afterMaintenance.test.ts
@@ -0,0 +1,75 @@
+// Copyright 2019-2022 Unique Network (Gibraltar) Ltd.
+// This file is part of Unique Network.
+
+// Unique Network is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Unique Network is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Unique Network. If not, see <http://www.gnu.org/licenses/>.
+
+import {IKeyringPair} from '@polkadot/types/types';
+import {ApiPromise} from '@polkadot/api';
+import {expect, itSub, Pallets, requirePalletsOrSkip, usingPlaygrounds} from '../../util';
+import {main as testedScript} from './correctStateAfterMaintenance';
+
+async function maintenanceEnabled(api: ApiPromise): Promise<boolean> {
+  return (await api.query.maintenance.enabled()).toJSON() as boolean;
+}
+
+
+
+describe('Integration Test: Maintenance mode & App Promo', () => {
+  let superuser: IKeyringPair;
+
+  before(async function() {
+    await usingPlaygrounds(async (helper, privateKey) => {
+      requirePalletsOrSkip(this, helper, [Pallets.Maintenance]);
+      superuser = await privateKey('//Alice');
+    });
+  });
+
+  describe('Test AppPromo script for check state after Maintenance mode', () => {
+    before(async function () {
+      await usingPlaygrounds(async (helper) => {
+        if(await maintenanceEnabled(helper.getApi())) {
+          console.warn('\tMaintenance mode was left enabled BEFORE the test suite! Disabling it now.');
+          await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.disable', [])).to.be.fulfilled;
+        }
+      });
+    });
+    itSub('Can find and fix inconsistent state', async({helper}) => {
+      const api = helper.getApi();
+
+      await helper.executeExtrinsic(superuser, 'api.tx.sudo.sudo', [api.tx.system.setStorage([
+        ['0x42b67acb8bd223c60d0c8f621ffefc0ae280fa2db99bd3827aac976de75af95f5153cb1f00942ff401000000',
+          '0x04d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d000010632d5ec76b0500000000000000'],
+        ['0x42b67acb8bd223c60d0c8f621ffefc0ae280fa2db99bd3827aac976de75af95f9eb2dcce60f37a2702000000',
+          '0x04d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d000010632d5ec76b0500000000000000'],
+        ['0xc2261276cc9d1f8598ea4b6a74b15c2fb1c0eb12e038e5c7f91e120ed4b7ebf1de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d',
+          '0x046170707374616b656170707374616b65000020c65abc8ed70a00000000000000'],
+      ])]);
+
+      // const pendingUnstaked = await helper.staking.getPendingUnstakePerBlock()
+      expect((await api.query.appPromotion.pendingUnstake(1)).toJSON()).to.be.deep.equal([[helper.address.normalizeSubstrateToChainFormat('5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'), '0x00000000000000056bc75e2d63100000']]);
+      expect((await api.query.appPromotion.pendingUnstake(2)).toJSON()).to.be.deep.equal([[helper.address.normalizeSubstrateToChainFormat('5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY'), '0x00000000000000056bc75e2d63100000']]);
+      await testedScript();
+
+      expect((await api.query.appPromotion.pendingUnstake(1)).toJSON()).to.be.deep.equal([]);
+      expect((await api.query.appPromotion.pendingUnstake(2)).toJSON()).to.be.deep.equal([]);
+
+    });
+
+    itSub('(!negative test!) Only works when Maintenance mode is disabled', async({helper}) => {
+      await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.enable', [])).to.be.fulfilled;
+      await expect(testedScript()).to.be.rejectedWith('The network is still in maintenance mode');
+      await expect(helper.getSudo().executeExtrinsic(superuser, 'api.tx.maintenance.disable', [])).to.be.fulfilled;
+    });
+  });
+});

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/collectData.ts
@@ -0,0 +1,12 @@
+import {exec} from 'child_process';
+import path from 'path';
+import {dirname} from 'path';
+import {fileURLToPath} from 'url';
+
+export const collectData = () => {
+  const dirName = dirname(fileURLToPath(import.meta.url));
+
+  const pathToScript = path.resolve(dirName, './stakersParser.jsonnet');
+  const outputPath = path.resolve(dirName, './output.json');
+  exec(`chainql --tla-str=chainUrl=ws://127.0.0.1:9944 ${pathToScript} > ${outputPath}`);
+};

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/correctStateAfterMaintenance.ts
@@ -0,0 +1,72 @@
+import {usingPlaygrounds} from '../../util';
+
+
+
+const WS_ENDPOINT = 'ws://localhost:9944';
+const DONOR_SEED = '//Alice';
+
+export const main = async(options: { wsEndpoint: string; donorSeed: string } = {
+  wsEndpoint: WS_ENDPOINT,
+  donorSeed: DONOR_SEED,
+}) => {
+  await usingPlaygrounds(async (helper, privateKey) => {
+    const api = helper.getApi();
+
+    if((await api.query.maintenance.enabled()).valueOf()) {
+      throw Error('The network is still in maintenance mode');
+    }
+
+    const pendingBlocks = (
+      await api.query.appPromotion.pendingUnstake.entries()
+    ).map(([k, _v]) =>
+      k.args[0]);
+
+    const currentBlock = await api.query.system.number();
+
+    const filteredBlocks = pendingBlocks.filter((b) => currentBlock.gt(b));
+
+    if(filteredBlocks.length != 0) {
+      console.log(
+        'During maintenance mode, %d block(s) were not processed',
+        filteredBlocks.length,
+      );
+    } else {
+      console.log('Nothing to change');
+      return;
+    }
+
+    const skippedBlocks = chunk(filteredBlocks, 10);
+
+    const signer = await privateKey(options.donorSeed);
+
+    const txs = skippedBlocks.map((b) =>
+      api.tx.sudo.sudo(api.tx.appPromotion.forceUnstake(b)));
+
+
+    const promises = txs.map((tx) => () => helper.signTransaction(signer, tx));
+
+    await Promise.allSettled(promises.map((p) => p()));
+
+    const failedBlocks: bigint[] = [];
+    let isSuccess = true;
+
+    for(const b of filteredBlocks) {
+      if(((await api.query.appPromotion.pendingUnstake(b)).toJSON() as any[]).length != 0) {
+        failedBlocks.push(b.toBigInt());
+        isSuccess = false;
+      }
+    }
+
+    if(isSuccess) {
+      console.log('Done. %d block(s) were processed.', filteredBlocks.length);
+    } else {
+      throw new Error(`Something went wrong. Block(s) have not been processed: ${failedBlocks}`);
+    }
+
+
+  }, options.wsEndpoint);
+};
+
+const chunk = <T>(arr: T[], size: number) =>
+  Array.from({length: Math.ceil(arr.length / size)}, (_: any, i: number) =>
+    arr.slice(i * size, i * size + size));
\ No newline at end of file

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/executeMigration.ts
@@ -0,0 +1,11 @@
+import {migrateLockedToFreeze} from './lockedToFreeze';
+
+
+const WS_RPC = process.env.WS_RPC || 'wss://ws-opal.unique.network:443';
+const SUPERUSER_SEED = process.env.SUPERUSER_SEED || '';
+
+migrateLockedToFreeze({
+  wsEndpoint: WS_RPC,
+  donorSeed: SUPERUSER_SEED,
+})
+  .catch(console.error);
\ No newline at end of file

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/index.ts
@@ -0,0 +1,12 @@
+import {Migration} from '../../util/frankensteinMigrate';
+import {collectData} from './collectData';
+import {migrateLockedToFreeze} from './lockedToFreeze';
+
+export const migration: Migration = {
+  async before() {
+    await collectData();
+  },
+  async after() {
+    await migrateLockedToFreeze();
+  },
+};

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/lockedToFreeze.ts
@@ -0,0 +1,258 @@
+// import { usingApi, privateKey, onlySign } from "./../../load/lib";
+import * as fs from 'fs';
+import {usingPlaygrounds} from '../../util';
+import path, {dirname} from 'path';
+import {isInteger, parse} from 'lossless-json';
+import {fileURLToPath} from 'url';
+
+
+const WS_ENDPOINT = 'ws://localhost:9944';
+const DONOR_SEED = '//Alice';
+const UPDATE_IF_VERSION = 942057;
+
+export function customNumberParser(value: any) {
+  return isInteger(value) ? BigInt(value) : parseFloat(value);
+}
+
+export const migrateLockedToFreeze = async(options: { wsEndpoint: string; donorSeed: string } = {
+  wsEndpoint: WS_ENDPOINT,
+  donorSeed: DONOR_SEED,
+}) => {
+  await usingPlaygrounds(async (helper, privateKey) => {
+    const api = helper.getApi();
+    // 1. Check version equal 942057 or skip
+    console.log((api.consts.system.version as any).specVersion.toNumber());
+    if((api.consts.system.version as any).specVersion.toNumber() != UPDATE_IF_VERSION) {
+      console.log("Version isn't 942057.");
+      return;
+    }
+
+    // 2. Get sudo signer
+    const signer = await privateKey(options.donorSeed);
+    console.log('2. Getting sudo:', signer.address);
+
+    // 3. Parse data to migrate
+    console.log('3. Parsing chainql results...');
+    const dirName = dirname(fileURLToPath(import.meta.url));
+    const parsingResult = parse(fs.readFileSync(path.resolve(dirName, 'output.json'), 'utf-8'), undefined, customNumberParser);
+
+    const chainqlImportData = parsingResult as {
+      address: string;
+      balance: string;
+      account: {
+        fee_frozen: string,
+        free: string,
+        misc_frozen: string,
+        reserved: string,
+      },
+      locks: {
+        amount: string,
+        id: string,
+      }[],
+      stakes: object,
+      unstakes: object,
+    }[];
+    testChainqlData(chainqlImportData);
+
+    const stakers = chainqlImportData.map((i) => i.address);
+
+    // 3.1 Split into chunks by 100
+    console.log('3.1 Splitting into chunks...');
+    const stakersChunks = chunk(stakers, 100);
+    console.log('3.1 Done, total chunks:', stakersChunks.length);
+
+    // 4. Get signer/sudo nonce
+    console.log('4. Getting sudo nonce...');
+    const signerAccount = (
+        await api.query.system.account(signer.address)
+      ).toJSON() as any;
+
+    let nonce: number = signerAccount.nonce;
+    console.log('4. Sudo nonce is:', nonce);
+
+    // 5. Only sign upgradeAccounts-transactions for each chunk
+    console.log('5. Signing transactions...');
+    const signedTxs = [];
+    for(const chunk of stakersChunks) {
+      const tx = api.tx.sudo.sudo(api.tx.appPromotion.upgradeAccounts(chunk));
+      const signed = tx.sign(signer, {
+        blockHash: api.genesisHash,
+        genesisHash: api.genesisHash,
+        runtimeVersion: api.runtimeVersion,
+        nonce: nonce++,
+      });
+      signedTxs.push(signed);
+    }
+
+    // 6. Send all signed transactions
+    console.log('6. Sending transactions...');
+    const promises = signedTxs.map((tx) => api.rpc.author.submitAndWatchExtrinsic(tx));
+    // 6.1 Wait all transactions settled
+    console.log('6.1 Waiting all transactions settled...');
+    const res = await Promise.allSettled(promises);
+
+    console.log('Wait 5 blocks for transactions to be included in a block...');
+    await helper.wait.newBlocks(5);
+    // 6.2 Filter failed transactions
+    console.log('6.2 Getting failed transactions...');
+    const failedTx = res.filter((r) => r.status == 'rejected') as PromiseRejectedResult[];
+    console.log('6.2. total failedTxs:', failedTx.length);
+
+    // 6.3 Log the reasons of failed tx
+    for(const tx of failedTx) {
+      console.log(tx.reason);
+    }
+
+    // 7. Check balances for 10 blocks:
+    console.log('7. Check balances...');
+    let blocksLeft = 10;
+    let notMigrated = stakers;
+    const suspiciousAccounts = [];
+    do {
+      console.log('blocks left:', blocksLeft);
+      const _notMigrated: string[] = [];
+      console.log('accounts to migrate...', notMigrated.length);
+      for(const accountToMigrate of notMigrated) {
+        let accountMigrated = true;
+        // 7.0 get data from chainql:
+        const oldAccount = chainqlImportData.find(acc => acc.address === accountToMigrate);
+        if(!oldAccount) {
+          console.log('Cannot find old account data for', accountMigrated);
+          accountMigrated = false;
+          _notMigrated.push(accountToMigrate);
+          continue;
+        }
+
+        // 7.1 system.account
+        const balance = await api.query.system.account(accountToMigrate) as any;
+        // new balances
+        const free = balance.data.free;
+        const reserved = balance.data.reserved;
+        const frozen = balance.data.frozen;
+        // old balances
+        const oldFree = oldAccount.account.free;
+        const oldReserved = oldAccount.account.reserved;
+        const oldFrozen = oldAccount.account.fee_frozen;
+        // asserts new = old
+        if(oldFree.toString() !== free.toString()) {
+          console.log('Old free !== New free, which is probably not a problem', oldFree.toString(), free.toString());
+          suspiciousAccounts.push(accountToMigrate);
+        }
+        if(oldFrozen.toString() !== frozen.toString()) {
+          console.log('Old frozen !== New frozen, which is probably not a problem', oldFrozen.toString(), frozen.toString());
+          suspiciousAccounts.push(accountToMigrate);
+        }
+        if(oldReserved.toString() !== reserved.toString()) {
+          console.log('Old reserved !== New reserved, which is probably not a problem', oldReserved.toString(), reserved.toString());
+          suspiciousAccounts.push(accountToMigrate);
+        }
+
+        // 7.2 balances.locks: no id appstake
+        const locks = await helper.balance.getLocked(accountToMigrate);
+        const appPromoLocks = locks.filter(lock => lock.id === 'appstake');
+        if(appPromoLocks.length !== 0) {
+          console.log('Account still has app-promo lock');
+          accountMigrated = false;
+        }
+
+        // 7.3 balances.freezes set...
+        let freezes = await api.query.balances.freezes(accountToMigrate) as any;
+        freezes = freezes.map((freez: any) => ({id: freez.id.toString(), amount: freez.amount.toString()}));
+        if(!freezes) {
+          console.log('Account does not have freezes');
+          accountMigrated = false;
+        } else {
+          const oldAppPromoLocks = oldAccount.locks.filter(l => l.id === '0x6170707374616b65'); // get app promo locks
+          // should be only one freez for each account
+          if(freezes.length !== 1) {
+            console.log('freezes.length !== 1 and old appPromoLocks.length', freezes.length, oldAppPromoLocks.length);
+            accountMigrated = false;
+          } else {
+            const appPromoFreez = freezes[0];
+            // freez-amount should be equal to migrated lock amount
+            if(appPromoFreez.amount.toString() !== oldAppPromoLocks[0].amount.toString()) {
+              console.log('freezes amount !== old appPromoLocks amount', appPromoFreez.amount.toString(), oldAppPromoLocks[0].amount.toString());
+              accountMigrated = false;
+            }
+            // freez id should be correct
+            if(appPromoFreez.id !== '0x6170707374616b656170707374616b65') {
+              console.log('Got freez with incorrect id:', appPromoFreez.id);
+              accountMigrated = false;
+            }
+          }
+        }
+
+        // 7.4 Stakes number the same
+        const stakesNumber = await helper.staking.getStakesNumber({Substrate: accountToMigrate});
+        const oldStakesNumber = oldAccount.stakes ? Object.keys(oldAccount.stakes).length : 0;
+        if(stakesNumber.toString() !== oldStakesNumber.toString()) {
+          console.log('Old stakes number !== New stakes number', oldStakesNumber, stakesNumber);
+          accountMigrated = false;
+        }
+
+        // 7.5 Total pendingUnstake + total staked = old locked
+        const pendingUnstakes = await helper.staking.getPendingUnstake({Substrate: accountToMigrate});
+        const totalStaked = await helper.staking.getTotalStaked({Substrate: accountToMigrate});
+        const totalBalanceInAppPromo = pendingUnstakes + totalStaked;
+        if(totalBalanceInAppPromo.toString() !== oldAccount.balance.toString()) {
+          console.log('totalBalanceInAppPromo !== old locked in app promo', totalBalanceInAppPromo.toString(), oldAccount.balance.toString());
+          accountMigrated = false;
+        }
+
+        // 8 Add to not-migrated
+        if(!accountMigrated) {
+          console.log('Add to not migrated:', accountToMigrate);
+          _notMigrated.push(accountToMigrate);
+        }
+      }
+
+      blocksLeft--;
+      notMigrated = _notMigrated;
+      await helper.wait.newBlocks(1);
+    } while(blocksLeft > 0 && notMigrated.length !== 0);
+
+    console.log('Not migrated accounts...', notMigrated.length);
+    if(suspiciousAccounts.length > 0) {
+      console.log('Saving suspicious accounts to suspicious.json:');
+      fs.writeFileSync('./suspicious.json', JSON.stringify(suspiciousAccounts));
+    }
+    if(notMigrated.length > 0) {
+      console.log('Saving not migrated list to failed.json:');
+      notMigrated.forEach(console.log);
+      fs.writeFileSync('./failed.json', JSON.stringify(notMigrated));
+      process.exit(1);
+    } else {
+      console.log('Migration success');
+    }
+  }, options.wsEndpoint);
+};
+
+const chunk = <T>(arr: T[], size: number) =>
+  Array.from({length: Math.ceil(arr.length / size)}, (_: any, i: number) =>
+    arr.slice(i * size, i * size + size));
+
+const testChainqlData = (data: any) => {
+  const wrongData = [];
+  for(const account of data) {
+    try {
+      if(account.address == null) throw Error('no address in data');
+      if(account.balance == null) throw Error('no balance in data');
+      if(account.account == null) throw Error('no account in data');
+      if(account.account.fee_frozen == null) throw Error('no account.fee_frozen in data');
+      if(account.account.misc_frozen == null) throw Error('no account.misc_frozen in data');
+      if(account.account.free == null) throw Error('no account.free in data');
+      if(account.account.reserved == null) throw Error('no account.reserved in data');
+      if(account.locks == null) throw Error('no locks in data');
+      if(account.locks[0].amount == null) throw Error('no locks.amount in data');
+      if(account.locks[0].id == null) throw Error('no locks.id in data');
+    } catch (error) {
+      wrongData.push(account.address);
+      console.log((error as Error).message, account.address);
+    }
+    if(wrongData.length > 0) {
+      console.log(data);
+      throw Error('Chainql data not correct');
+    }
+  }
+  console.log('Chainql data correct');
+};

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/runCheckState.ts
@@ -0,0 +1,13 @@
+import {main} from './correctStateAfterMaintenance';
+
+
+
+
+main({
+  wsEndpoint: process.env.WS_RPC!,
+  donorSeed: process.env.SUPERUSER_SEED!,
+}).then(() => process.exit(0))
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });
\ No newline at end of file

--- /dev/null
+++ b/tests/src/migrations/942057-appPromotion/stakersParser.jsonnet
@@ -0,0 +1,29 @@
+function(chainUrl)
+
+  local
+    state = cql.chain(chainUrl).latest,
+    locked_balances = state.Balances.Locks._preloadKeys,
+    accountsRaw = state.System.Account._preloadKeys,
+    stakers = state.AppPromotion.Staked._preloadKeys,
+    unstakes = state.AppPromotion.PendingUnstake._preloadKeys,
+    locks = [
+      { [k]: std.filter(function(l) l.id == '0x6170707374616b65', locked_balances[k]) }
+      for k in std.objectFields(locked_balances)
+    ],
+    unstakersData = [
+      { [pair[0]]: { block: block, value: pair[1] } }
+
+      for block in std.objectFields(unstakes)
+      for pair in unstakes[block]
+    ],
+    non_empty_locks = std.prune(locks)
+  ;
+
+  std.map(function(a) {
+    address: std.objectFields(a)[0],
+    balance: a[std.objectFields(a)[0]][0].amount,
+    account: accountsRaw[std.objectFields(a)[0]].data,
+    locks: locked_balances[std.objectFields(a)[0]],
+    stakes: if std.objectHas(stakers, std.objectFields(a)[0]) then stakers[std.objectFields(a)[0]] else null,
+    unstakes: std.filterMap(function(b) std.objectHas(b, std.objectFields(a)[0]), function(c) std.objectValues(c)[0], unstakersData),
+  }, non_empty_locks)

--- /dev/null
+++ b/tests/src/util/frankensteinMigrate.ts
@@ -0,0 +1,9 @@
+import {migration as locksToFreezesMigration} from '../migrations/942057-appPromotion';
+export interface Migration {
+  before: () => Promise<void>,
+  after: () => Promise<void>,
+}
+
+export const migrations: {[key: string]: Migration} = {
+  'v942057': locksToFreezesMigration,
+};