From b4c2a0c88d0a2e0f7f166e07a240c8137e10aae4 Mon Sep 17 00:00:00 2001 From: Josip Milovac Date: Tue, 20 Dec 2022 11:26:06 +1100 Subject: [PATCH] smarter store update after replacing chains concurrent mining validity checks for new transactions/blocks/metadata rdf store stores quads smarter clearing of transactions from pool when a new block is mined change querying to be readonly and generic moved some things around --- app/index.js | 146 ++++++++++++++++++++----------------- app/miner.js | 84 +++++++++++++++++---- app/p2p-server.js | 126 ++++++++++++++++++++++++-------- blockchain/block.js | 41 ++++++----- blockchain/index.js | 49 +++++++++++-- wallet/index.js | 38 +++------- wallet/transaction-pool.js | 66 ++++++++++++++--- 7 files changed, 373 insertions(+), 177 deletions(-) diff --git a/app/index.js b/app/index.js index b000015..ca2d326 100644 --- a/app/index.js +++ b/app/index.js @@ -33,8 +33,8 @@ const Blockchain = require('../blockchain'); const P2pServer = require('./p2p-server'); const Wallet = require('../wallet'); const TransactionPool = require('../wallet/transaction-pool'); -const Miner = require('./miner'); const QueryEngine = require('@comunica/query-sparql').QueryEngine; +const ChainUtil = require('../chain-util'); const N3 = require('n3'); const jsonld = require('jsonld'); @@ -51,10 +51,10 @@ const multer = require('multer');/* Multer is a node.js middleware for const app = express(); const bc = new Blockchain(); -const wallet = new Wallet(); +//currently gen a new keypair per run, we probably want to load this from something else in the future +const wallet = new Wallet(ChainUtil.genKeyPair()); const tp = new TransactionPool(); -const p2pServer = new P2pServer(bc, tp,'./persist_block_chain.json'); -const miner = new Miner(bc, tp, wallet, p2pServer); +const p2pServer = new P2pServer(bc, tp, wallet, './persist_block_chain.json'); const parser = new N3.Parser(); //({format: 'application/n-quads'}); const myEngine = new QueryEngine(); @@ -119,12 +119,12 @@ app.get('/Transactions', (req, res) => { res.json(tp); }); /////////////// -app.get('/mine-transactions', (req, res) => { - const block = miner.mine(); - console.log(`New block added: ${block.toString()}`); - res.redirect('/blocks'); - // res.json("Block mined"); -}); +//app.get('/mine-transactions', (req, res) => { +// const block = miner.mine(); +// console.log(`New block added: ${block.toString()}`); +// res.redirect('/blocks'); +// // res.json("Block mined"); +//}); /////////////// app.get('/public-key', (req, res) => { res.json({ publicKey: wallet.publicKey }); @@ -137,60 +137,66 @@ app.get('/Balance', (req, res) => { /////////////// //this API prints all the quads stored in the RDF store and returns the entire store app.get('/quads', (req, res) => { - for (const quad of store) - console.log(quad); + //for (const quad of store) + //console.log(quad); res.json(store); }); app.get('/IoTdeviceRegistration', (req, res)=> { fs.readdir('./uploads', function(err, files) { - console.log(files[files.length-2]); - var FileName = files[files.length-2]; - let rawdata = fs.readFileSync(`./uploads/${FileName}`); - let SenShaMartDesc = JSON.parse(rawdata); -/* the following piece of code is used to genrate JSON object out of name-value pairs submitted - let SenShaMartExtNames = ['Name','Geo' ,'IP_URL' , 'Topic_Token', 'Permission', 'RequestDetail', - 'OrgOwner', 'DepOwner','PrsnOwner', 'PaymentPerKbyte', - 'PaymentPerMinute','Protocol', 'MessageAttributes', 'Interval', - 'FurtherDetails'] - let SenShaMartExtValues = [Name,Geo ,IP_URL , Topic_Token, Permission, RequestDetail, - OrgOwner, DepOwner,PrsnOwner, PaymentPerKbyte, - PaymentPerMinute,Protocol, MessageAttributes, Interval, - FurtherDetails] - let SenSHaMArtExt = {}; - for (let i =0; i { - console.log(nquads) - var metaDataTransaction = wallet.createMetadata( - nquads, tp); + } + //let SenShaMartOnt = SSNmetadata; + //SenShaMartOnt.push(SenSHaMArtExt); */ + //console.log(SenShaMartDesc); + jsonld.toRDF(SenShaMartDesc, {format: 'application/n-quads'}, + (err, nquads) => { + //console.log(nquads) + var metadata = wallet.createMetadata( + nquads, tp); + p2pServer.newMetadata(metadata); + }); }); + res.json("MetadataTransactionCreated"); }); - res.json("MetadataTransactionCreated"); -}); ////////////////////////////////////////////////// // POST APIs -app.post('/mine', (req, res) => { - const block = bc.addBlock(req.body.data); - console.log(`New block added: ${block.toString()}`); +//this doesn't work well with the continious miner +//app.post('/mine', (req, res) => { +// const block = bc.addBlock(req.body.data); +// console.log(`New block added: ${block.toString()}`); - p2pServer.newBlock(block); +// p2pServer.newBlock(block); - res.redirect('/blocks'); -}); +// res.redirect('/blocks'); +//}); /////////////// app.post('/PaymentTransaction', (req, res) => { const { recipient, amount } = req.body; const transaction = wallet.createTransaction(recipient, amount, bc, tp); - //p2pServer.broadcastTransaction(transaction); + if (transaction === null) { + res.json("Couldn't create transaction"); + return; + } + p2pServer.newTransaction(transaction); res.redirect('/transactions'); }); @@ -198,11 +204,12 @@ app.post('/PaymentTransaction', (req, res) => { app.post('/IoTdevicePaymentTransaction', (req, res) => { const { Recipient_payment_address, Amount_of_money, Payment_method, Further_details} = req.body; - if (Payment_method == "SensorCoin"){ - const PaymentTransaction = wallet.createCoinTransaction( - Recipient_payment_address, Amount_of_money, bc, tp); - p2pServer.broadcastCoinTransaction(PaymentTransaction); - res.json("PaymentTransactionCreated"); + if (Payment_method == "SensorCoin") { + //create coin transaction doesn't exist yet + const PaymentTransaction = wallet.createCoinTransaction( + Recipient_payment_address, Amount_of_money, bc, tp); + p2pServer.broadcastCoinTransaction(PaymentTransaction); + res.json("PaymentTransactionCreated"); } else if (Payment_method == "Bitcoin") { res.redirect('/BitcoinTransaction') @@ -233,21 +240,30 @@ app.post("/UploadMetafile", upload.single('file'), (req, res) => { //Start of comunica sparql query code app.post('/sparql', (req, res) => { console.log(req.body); - const {Select,subject,predicate,object,Limit}= req.body; - const start = async function (a,b){ - const bindingsStream = await myEngine.queryBindings(`SELECT ${Select} WHERE - {${subject} ${predicate} ${object}} LIMIT - ${Limit}`, { sources: [{ type: 'rdfjsSource', - value: p2pServer.store}] - }); - bindingsStream.on('data', (binding) => { - console.log(binding.toString()); - queryResult= binding; + const start = async function () { + let result = []; + const bindingsStream = await myEngine.queryBindings( + req.body, + { + readOnly: true, + sources: [{ + type: 'rdfjsSource', + value: p2pServer.store + }] }); - }; - start() - res.json("Query succsessful"); - }); + bindingsStream.on('data', (binding) => { + console.log(binding.toString()); + result.push(binding); + }); + bindingsStream.on('end', () => { + res.json(JSON.stringify(result)); + }); + bindingsStream.on('error', (err) => { + console.error(err); + }); + }; + start() +}); ///////////////////////////////////////////////////////////Integration/////////////////////////////////////////////////////////// DistributedBrokers = ["mqtt.eclipse.org", "test.mosquitto.org","broker.hivemq.com"]; diff --git a/app/miner.js b/app/miner.js index b5a530e..ad053e8 100644 --- a/app/miner.js +++ b/app/miner.js @@ -1,34 +1,88 @@ const Wallet = require('../wallet'); const Transaction = require('../wallet/transaction'); +const Block = require('../blockchain/block'); class Miner { + static STATE_WAITING = 0; + static STATE_RUNNING = 1; + static STATE_INTERRUPTED = 2; + static STATE_RESTARTING = 3; + constructor(blockchain, transactionPool, wallet, p2pServer) { this.blockchain = blockchain; this.transactionPool = transactionPool; this.wallet = wallet; this.p2pServer = p2pServer; - } + this.state = Miner.STATE_WAITING; + this.mining = [[], []]; + this.lastBlock = null; + } + + interrupt() { + if (this.state === Miner.STATE_RUNNING) { + this.state = Miner.STATE_INTERRUPTED; + } + } + + interruptIfContainsTransaction(transaction) { + if (this.state === Miner.STATE_RUNNING && this.mining[0].find(t => t.id === transaction.id)) { + this.state = Miner.STATE_INTERRUPTED; + } + } + interruptIfContainsMetadata(metadata) { + if (this.state === Miner.STATE_RUNNING && this.mining[1].find(t => t.id === metadata.id)) { + this.state = Miner.STATE_INTERRUPTED; + } + } + + startMine() { + //only continue if state is waiting or restarting + if (this.state !== Miner.STATE_WAITING && this.state !== Miner.STATE_RESTARTING) { + return; + } - mine() { const validTransactions = this.transactionPool.validTransactions(); + const validMetadataS = this.transactionPool.validMetadataS(); + + if (validTransactions.length === 0 && validMetadataS.length === 0) { + this.state = Miner.STATE_WAITING; + return; + } + validTransactions.push( Transaction.rewardTransaction(this.wallet, Wallet.blockchainWallet()) ); - console.log(validTransactions); - console.log("//////"); - const validMetadataS = this.transactionPool.validMetadataS(); - // for (let i =0; i { this.startMine() }); + } else { + //failure + this.nonce++; + setImmediate(() => { this.mine() }); + } } } diff --git a/app/p2p-server.js b/app/p2p-server.js index 2b9bcf4..e0f0a88 100644 --- a/app/p2p-server.js +++ b/app/p2p-server.js @@ -1,9 +1,12 @@ const Websocket = require('ws'); const N3 = require('n3'); -const parser = new N3.Parser(); //({format: 'application/n-quads'}); const DataFactory = require('n3').DataFactory; const fs = require('fs'); const process = require('process'); +const Miner = require('./miner'); +const Transaction = require('../wallet/transaction'); +const TransactionPool = require('../wallet/transaction-pool'); +const Metadata = require('../wallet/metadata'); const P2P_PORT = process.env.P2P_PORT || 5000; const peers = process.env.PEERS ? process.env.PEERS.split(',') : []; @@ -15,12 +18,13 @@ const MESSAGE_TYPES = { }; class P2pServer { - constructor(blockchain, transactionPool,chainStorageLocation) { + constructor(blockchain, transactionPool, wallet, chainStorageLocation) { this.blockchain = blockchain; this.transactionPool = transactionPool; this.sockets = []; this.store = new N3.Store(); this.chainStorageLocation = chainStorageLocation; + this.miner = new Miner(this.blockchain, this.transactionPool, wallet, this); //possible race if deleted after check, but we live with it I guess if (fs.existsSync(this.chainStorageLocation)) { @@ -63,39 +67,93 @@ class P2pServer { const data = JSON.parse(message); switch(data.type) { case MESSAGE_TYPES.chain: - newChain(data.chain); + this.newChain(data.chain); break; case MESSAGE_TYPES.transaction: - this.transactionPool.updateOrAddTransaction(data.transaction); + this.newTransaction(data.transaction, false); break; case MESSAGE_TYPES.metadata: - this.transactionPool.updateOrAddMetadata(data.metadata); - break; - case MESSAGE_TYPES.clear_transactions: - this.transactionPool.clear(); + this.newMetadata(data.metadata, false); break; + //case MESSAGE_TYPES.clear_transactions: + // this.transactionPool.clear(); + // break; } }); } - newBlock(block) { - this.onNewBlock(block.data); - this.syncChains(); - this.persistChain(this.blockchain.chain); + newMetadata(metadata, broadcast) { + if (!Metadata.verifyMetadata(metadata)) { + console.log("Couldn't add metadata to p2pServer, couldn't verify"); + return; + } + + switch (this.transactionPool.updateOrAddMetadata(metadata)) { + case TransactionPool.Return.add: + this.miner.startMine(); + break; + case TransactionPool.Return.update: + this.miner.interruptIfContainsMetadata(metadata); + break; + case TransactionPool.Return.error: + console.log("Couldn't add metadata to p2pServer, couldn't updateOrAdd"); + return; + } + + if (broadcast === undefined || broadcast) { + this.broadcastMetadata(metadata); + } } - newChain(chain,persist) { - if (!this.blockchain.replaceChain(chain)) { + newTransaction(transaction, broadcast) { + if (!Transaction.verifyTransaction(transaction)) { + console.log("Couldn't add transaction to p2pServer, couldn't verify"); + return false; + } + + switch (this.transactionPool.updateOrAddTransaction(transaction)) { + case TransactionPool.Return.add: + this.miner.startMine(); + break; + case TransactionPool.Return.update: + this.miner.interruptIfContainsTransaction(transaction); + break; + case TransactionPool.Return.error: + console.log("Couldn't add transaction to p2pServer, couldn't updateOrAdd"); + return; + } + + if (broadcast === undefined || broadcast) { + this.broadcastTransaction(transaction); + } + } + + newBlock(block) { + if (!this.blockchain.addBlock(block)) { + //invalid block, return + return; + } + this.onNewBlock(block); + this.persistChain(this.blockchain.chain); + this.syncChains(); + } + + newChain(chain, persist) { + const oldChain = this.blockchain.chain; + const divergence = this.blockchain.replaceChain(chain); + + if (divergence === null) { //failed to replace return; } if (typeof persist === "undefined" || persist) { this.persistChain(chain); } - //dirty clear - this.store = new N3.Store(); - for (var block in this.blockchain.chain) { - this.onNewBlock(block); + for (let i = divergence; i < oldChain.length; i++) { + this.store.deleteGraph(oldChain[i].hash); + } + for (let i = divergence; i < this.blockchain.chain.length; i++) { + this.onNewBlock(this.blockchain.chain[i]); } } @@ -112,28 +170,36 @@ class P2pServer { onNewBlock(block) { //block data is of form [transactions,metadatas] - if (block.length != 2) { + if (block.data.length != 2) { //assert? return; } - const metadatas = block[1]; - for (var metadata in metadatas) { - if (!(SSNmetadata in metadata)) { + this.transactionPool.clearFromBlock(block); + + this.miner.interrupt(); + + const metadatas = block.data[1]; + + for (const metadata of metadatas) { + if (!("SSNmetadata" in metadata)) { //assert? return; } var ssn = metadata.SSNmetadata; + const parser = new N3.Parser(); + parser.parse( ssn, (error, quadN, prefixes) => { if (quadN) { - store.addQuad(DataFactory.quad( + this.store.addQuad(DataFactory.quad( DataFactory.namedNode(quadN.subject.id), DataFactory.namedNode(quadN.predicate.id), - DataFactory.namedNode(quadN.object.id))); + DataFactory.namedNode(quadN.object.id), + DataFactory.namedNode(block.hash))); } }); } @@ -163,13 +229,13 @@ class P2pServer { this.sockets.forEach(socket => this.sendChain(socket)); } - //broadcastTransaction(transaction) { - // this.sockets.forEach(socket => this.sendTransaction(socket, transaction)); - //} + broadcastTransaction(transaction) { + this.sockets.forEach(socket => this.sendTransaction(socket, transaction)); + } - //broadcastMetadata(metadata) { - // this.sockets.forEach(socket => this.sendMetadata(socket, metadata)); - //} + broadcastMetadata(metadata) { + this.sockets.forEach(socket => this.sendMetadata(socket, metadata)); + } //broadcastClearTransactions() { // this.sockets.forEach(socket => socket.send(JSON.stringify({ diff --git a/blockchain/block.js b/blockchain/block.js index 0f44ac8..7b227f1 100644 --- a/blockchain/block.js +++ b/blockchain/block.js @@ -8,7 +8,11 @@ class Block { this.hash = hash; this.data = data; this.nonce = nonce; - this.difficulty = difficulty || DIFFICULTY; + if (difficulty === undefined) { + this.difficulty = DIFFICULTY; + } else { + this.difficulty = difficulty; + } } toString() { @@ -24,26 +28,20 @@ class Block { static genesis() { return new this('Genesis time', '-----', 'f1r57-h45h', [], 0, DIFFICULTY); } - //we want this to eventually be continously running where there are things in the pool, - //however as node is single threaded, this almost has to be a fiber, and yield after every - //other iteration to allow for meaningful forward progress - //we can either add all new transactions into the block as we see them, or stay with the starting list, idk which - //to be done later - static mineBlock(lastBlock, data) { - let hash, timestamp; - const lastHash = lastBlock.hash; - let { difficulty } = lastBlock; - let nonce = 0; + //returns false if hash doesn't match + static checkHash(hash, timestamp, lastHash, data, nonce, difficulty) { + const computedHash = Block.hash(timestamp, lastHash, data, nonce, difficulty); - do { - nonce++; - timestamp = Date.now(); - difficulty = Block.adjustDifficulty(lastBlock, timestamp); - hash = Block.hash(timestamp, lastHash, data, nonce, difficulty); - } while (hash.substring(0, difficulty) !== '0'.repeat(difficulty)); + if (computedHash !== hash) { + return false; + } - return new this(timestamp, lastHash, hash, data, nonce, difficulty); + if (hash.substring(0, difficulty) !== '0'.repeat(difficulty)) { + return false; + } + + return true; } static hash(timestamp, lastHash, data, nonce, difficulty) { @@ -55,11 +53,16 @@ class Block { return Block.hash(timestamp, lastHash, data, nonce, difficulty); } + //returns false if block's hash doesn't match internals + static checkBlock(block) { + return Block.checkHash(block.hash, block.timestamp, block.lastHash, block.data, block.nonce, block.difficulty); + } + static adjustDifficulty(lastBlock, currentTime) { let { difficulty } = lastBlock; difficulty = lastBlock.timestamp + MINE_RATE > currentTime ? difficulty + 1 : difficulty - 1; - return difficulty; + return Math.max(0, difficulty); } } diff --git a/blockchain/index.js b/blockchain/index.js index 84bffb3..f851175 100644 --- a/blockchain/index.js +++ b/blockchain/index.js @@ -5,15 +5,37 @@ class Blockchain { this.chain = [Block.genesis()]; } - addBlock(data) { - const block = Block.mineBlock(this.chain[this.chain.length-1], data); - this.chain.push(block); + //adds an existing block to the blockchain, returns false if the block can't be added, true if it was added + addBlock(newBlock) { + if (newBlock.lastHash !== this.chain[this.chain.length - 1].hash) { + console.log("Tried to add invalid block, last hash didn't match our last hash"); + return false; + } + //how to check if new block's timestamp is believable + if (newBlock.difficulty !== Block.adjustDifficulty(this.chain[this.chain.length - 1], newBlock.timestamp)) { + console.log("Tried to add invalid block, difficulty is incorrect"); + return false; + } + if (!Block.checkBlock(newBlock)) { + console.log("Tried to add invalid block, block's hash doesn't match its contents"); + return false; + } - return block; + this.chain.push(newBlock); + + console.log("Added new block: "); + //console.log(newBlock); + + return true; } isValidChain(chain) { - if(JSON.stringify(chain[0]) !== JSON.stringify(Block.genesis())) return false; + if (chain.length === 0) { + return false; + } + if (JSON.stringify(chain[0]) !== JSON.stringify(Block.genesis())) { + return false; + } for (let i=1; i this.balance) { console.log(`Amount: ${amount} exceceds current balance: ${this.balance}`); - return; + return null; } - let transaction = transactionPool.existingTransaction(this.publicKey); - - if (transaction) { - transaction.update(this, recipient, amount); - } else { - transaction = Transaction.newTransaction(this, recipient, amount); - transactionPool.updateOrAddTransaction(transaction); - } - - return transaction; + return Transaction.newTransaction(this, recipient, amount); } - createMetadata(SSNmetadata, transactionPool){ - //let metadata = transactionPool.existingMetadata(this.publicKey); + createMetadata(SSNmetadata) { + return Metadata.newMetadata(this, SSNmetadata); + } - // if (metaData) { - // metadata.update(this, Geo, Std, Name,MetaHash,file); - // } else {*/ - - let metadata= Metadata.newMetadata(this, SSNmetadata); - transactionPool.AddMetadata(metadata); - - //} - return metadata; - } - - calculateBalance(blockchain) { let balance = this.balance; let transactions = []; @@ -93,7 +73,7 @@ class Wallet { } static blockchainWallet() { - const blockchainWallet = new this(); + const blockchainWallet = new this(ChainUtil.genKeyPair()); blockchainWallet.address = 'blockchain-wallet'; return blockchainWallet; } diff --git a/wallet/transaction-pool.js b/wallet/transaction-pool.js index 08d5d57..71648f6 100644 --- a/wallet/transaction-pool.js +++ b/wallet/transaction-pool.js @@ -1,29 +1,50 @@ const Transaction = require('../wallet/transaction'); -const Metadata = require('../wallet/metadata') +const Metadata = require('../wallet/metadata') + +const Return = { + add: 1, + update: 2, + error: 3 +}; + class TransactionPool { constructor() { this.transactions = []; this.metadataS =[]; } + //returns true on update, false on add updateOrAddTransaction(transaction) { - let transactionWithId = this.transactions.find(t => t.id === transaction.id); + if (!Transaction.verifyTransaction(transaction)) { + console.log("Couldn't update or add transaction, transaction couldn't be verified"); + return Return.error; + } + const foundIndex = this.transactions.findIndex(t => t.id === transaction.id); - if (transactionWithId) { - this.transactions[this.transactions.indexOf(transactionWithId)] = transaction; + if (foundIndex !== -1) { + this.transactions[foundIndex] = transaction; + return Return.update; } else { this.transactions.push(transaction); + return Return.add; } } - AddMetadata(metadata) { - // let metadataWithId = this.metadataS.find(t => t.id === metadata.id); + updateOrAddMetadata(metadata) { + if (!Metadata.verifyMetadata(metadata)) { + console.log("Couldn't update metdata, metadata couldn't be verified"); + return Return.error; + } - // if (metadataWithId) { - // this.metaDataS[this.metadataS.indexOf(metadataWithId)] = metadata; - // } else { + const foundIndex = this.metadataS.findIndex(t => t.id === metadata.id); + + if (foundIndex !== -1) { + this.metadataS[foundIndex] = metadata; + return Return.update; + } else { this.metadataS.push(metadata); - // } + return Return.add; + } } existingTransaction(address) { @@ -64,10 +85,31 @@ class TransactionPool { }); } - clear() { + clearFromBlock(block) { + const transactions = block.data[0]; + const metadatas = block.data[1]; + for (const transaction of transactions) { + const foundTransaction = this.transactions.findIndex(t => t.id === transaction.id); + + if (foundTransaction !== -1) { + this.transactions.splice(foundTransaction, 1); + } + } + + for (const metadata of metadatas) { + const foundMetadata = this.metadataS.findIndex(m => m.id === metadata.id); + + if (foundMetadata !== -1) { + this.metadataS.splice(foundMetadata, 1); + } + } + } + + clearAll() { this.transactions = []; this.metadataS = []; } } -module.exports = TransactionPool; \ No newline at end of file +module.exports = TransactionPool; +module.exports.Return = Return; \ No newline at end of file