smarter store update after replacing chains

concurrent mining
validity checks for new transactions/blocks/metadata
rdf store stores quads
smarter clearing of transactions from pool when a new block is mined
change querying to be readonly and generic
moved some things around
This commit is contained in:
Josip Milovac 2022-12-20 11:26:06 +11:00
parent ea81105df6
commit b4c2a0c88d
7 changed files with 373 additions and 177 deletions

View file

@ -33,8 +33,8 @@ const Blockchain = require('../blockchain');
const P2pServer = require('./p2p-server');
const Wallet = require('../wallet');
const TransactionPool = require('../wallet/transaction-pool');
const Miner = require('./miner');
const QueryEngine = require('@comunica/query-sparql').QueryEngine;
const ChainUtil = require('../chain-util');
const N3 = require('n3');
const jsonld = require('jsonld');
@ -51,10 +51,10 @@ const multer = require('multer');/* Multer is a node.js middleware for
const app = express();
const bc = new Blockchain();
const wallet = new Wallet();
//currently gen a new keypair per run, we probably want to load this from something else in the future
const wallet = new Wallet(ChainUtil.genKeyPair());
const tp = new TransactionPool();
const p2pServer = new P2pServer(bc, tp,'./persist_block_chain.json');
const miner = new Miner(bc, tp, wallet, p2pServer);
const p2pServer = new P2pServer(bc, tp, wallet, './persist_block_chain.json');
const parser = new N3.Parser(); //({format: 'application/n-quads'});
const myEngine = new QueryEngine();
@ -119,12 +119,12 @@ app.get('/Transactions', (req, res) => {
res.json(tp);
});
///////////////
app.get('/mine-transactions', (req, res) => {
const block = miner.mine();
console.log(`New block added: ${block.toString()}`);
res.redirect('/blocks');
// res.json("Block mined");
});
//app.get('/mine-transactions', (req, res) => {
// const block = miner.mine();
// console.log(`New block added: ${block.toString()}`);
// res.redirect('/blocks');
// // res.json("Block mined");
//});
///////////////
app.get('/public-key', (req, res) => {
res.json({ publicKey: wallet.publicKey });
@ -137,19 +137,19 @@ app.get('/Balance', (req, res) => {
///////////////
//this API prints all the quads stored in the RDF store and returns the entire store
app.get('/quads', (req, res) => {
for (const quad of store)
console.log(quad);
//for (const quad of store)
//console.log(quad);
res.json(store);
});
app.get('/IoTdeviceRegistration', (req, res)=> {
fs.readdir('./uploads', function(err, files) {
console.log(files[files.length-2]);
//console.log(files[files.length-2]);
var FileName = files[files.length-2];
let rawdata = fs.readFileSync(`./uploads/${FileName}`);
let SenShaMartDesc = JSON.parse(rawdata);
/* the following piece of code is used to genrate JSON object out of name-value pairs submitted
/* the following piece of code is used to genrate JSON object out of name-value pairs submitted
let SenShaMartExtNames = ['Name','Geo' ,'IP_URL' , 'Topic_Token', 'Permission', 'RequestDetail',
'OrgOwner', 'DepOwner','PrsnOwner', 'PaymentPerKbyte',
'PaymentPerMinute','Protocol', 'MessageAttributes', 'Interval',
@ -163,34 +163,40 @@ app.get('/IoTdeviceRegistration', (req, res)=> {
SenSHaMArtExt[`${SenShaMartExtNames[i]}`]= SenShaMartExtValues[i]
}
//let SenShaMartOnt = SSNmetadata;
//SenShaMartOnt.push(SenSHaMArtExt); */
console.log(SenShaMartDesc)
//let SenShaMartOnt = SSNmetadata;
//SenShaMartOnt.push(SenSHaMArtExt); */
//console.log(SenShaMartDesc);
jsonld.toRDF(SenShaMartDesc, {format: 'application/n-quads'},
(err, nquads) => {
console.log(nquads)
var metaDataTransaction = wallet.createMetadata(
//console.log(nquads)
var metadata = wallet.createMetadata(
nquads, tp);
p2pServer.newMetadata(metadata);
});
});
res.json("MetadataTransactionCreated");
});
});
//////////////////////////////////////////////////
// POST APIs
app.post('/mine', (req, res) => {
const block = bc.addBlock(req.body.data);
console.log(`New block added: ${block.toString()}`);
//this doesn't work well with the continious miner
//app.post('/mine', (req, res) => {
// const block = bc.addBlock(req.body.data);
// console.log(`New block added: ${block.toString()}`);
p2pServer.newBlock(block);
// p2pServer.newBlock(block);
res.redirect('/blocks');
});
// res.redirect('/blocks');
//});
///////////////
app.post('/PaymentTransaction', (req, res) => {
const { recipient, amount } = req.body;
const transaction = wallet.createTransaction(recipient, amount, bc, tp);
//p2pServer.broadcastTransaction(transaction);
if (transaction === null) {
res.json("Couldn't create transaction");
return;
}
p2pServer.newTransaction(transaction);
res.redirect('/transactions');
});
@ -198,7 +204,8 @@ app.post('/PaymentTransaction', (req, res) => {
app.post('/IoTdevicePaymentTransaction', (req, res) => {
const { Recipient_payment_address, Amount_of_money, Payment_method,
Further_details} = req.body;
if (Payment_method == "SensorCoin"){
if (Payment_method == "SensorCoin") {
//create coin transaction doesn't exist yet
const PaymentTransaction = wallet.createCoinTransaction(
Recipient_payment_address, Amount_of_money, bc, tp);
p2pServer.broadcastCoinTransaction(PaymentTransaction);
@ -233,21 +240,30 @@ app.post("/UploadMetafile", upload.single('file'), (req, res) => {
//Start of comunica sparql query code
app.post('/sparql', (req, res) => {
console.log(req.body);
const {Select,subject,predicate,object,Limit}= req.body;
const start = async function (a,b){
const bindingsStream = await myEngine.queryBindings(`SELECT ${Select} WHERE
{${subject} ${predicate} ${object}} LIMIT
${Limit}`, { sources: [{ type: 'rdfjsSource',
value: p2pServer.store}]
const start = async function () {
let result = [];
const bindingsStream = await myEngine.queryBindings(
req.body,
{
readOnly: true,
sources: [{
type: 'rdfjsSource',
value: p2pServer.store
}]
});
bindingsStream.on('data', (binding) => {
console.log(binding.toString());
queryResult= binding;
result.push(binding);
});
bindingsStream.on('end', () => {
res.json(JSON.stringify(result));
});
bindingsStream.on('error', (err) => {
console.error(err);
});
};
start()
res.json("Query succsessful");
});
});
///////////////////////////////////////////////////////////Integration///////////////////////////////////////////////////////////
DistributedBrokers = ["mqtt.eclipse.org", "test.mosquitto.org","broker.hivemq.com"];

View file

@ -1,34 +1,88 @@
const Wallet = require('../wallet');
const Transaction = require('../wallet/transaction');
const Block = require('../blockchain/block');
class Miner {
static STATE_WAITING = 0;
static STATE_RUNNING = 1;
static STATE_INTERRUPTED = 2;
static STATE_RESTARTING = 3;
constructor(blockchain, transactionPool, wallet, p2pServer) {
this.blockchain = blockchain;
this.transactionPool = transactionPool;
this.wallet = wallet;
this.p2pServer = p2pServer;
this.state = Miner.STATE_WAITING;
this.mining = [[], []];
this.lastBlock = null;
}
interrupt() {
if (this.state === Miner.STATE_RUNNING) {
this.state = Miner.STATE_INTERRUPTED;
}
}
interruptIfContainsTransaction(transaction) {
if (this.state === Miner.STATE_RUNNING && this.mining[0].find(t => t.id === transaction.id)) {
this.state = Miner.STATE_INTERRUPTED;
}
}
interruptIfContainsMetadata(metadata) {
if (this.state === Miner.STATE_RUNNING && this.mining[1].find(t => t.id === metadata.id)) {
this.state = Miner.STATE_INTERRUPTED;
}
}
startMine() {
//only continue if state is waiting or restarting
if (this.state !== Miner.STATE_WAITING && this.state !== Miner.STATE_RESTARTING) {
return;
}
mine() {
const validTransactions = this.transactionPool.validTransactions();
const validMetadataS = this.transactionPool.validMetadataS();
if (validTransactions.length === 0 && validMetadataS.length === 0) {
this.state = Miner.STATE_WAITING;
return;
}
validTransactions.push(
Transaction.rewardTransaction(this.wallet, Wallet.blockchainWallet())
);
console.log(validTransactions);
console.log("//////");
const validMetadataS = this.transactionPool.validMetadataS();
// for (let i =0; i <validMetadataS.length; i++){
// validTransactions.push(validMetadataS[i]);
// }
console.log(validTransactions);
// const validMetadataS = this.transactionPool.metadataS;
const block = this.blockchain.addBlock([validTransactions, validMetadataS]);
this.p2pServer.newBlock(block);
this.transactionPool.clear();
this.lastBlock = this.blockchain.chain[this.blockchain.chain.length - 1];
return block;
this.state = Miner.STATE_RUNNING;
this.mining = [validTransactions, validMetadataS];
this.nonce = 0;
this.mine();
}
mine() {
if (this.state !== Miner.STATE_RUNNING) {
this.state = Miner.STATE_RESTARTING;
startMine();
return;
}
const timestamp = Date.now();
const difficulty = Block.adjustDifficulty(this.lastBlock, timestamp);
const hash = Block.hash(timestamp, this.lastBlock.hash, this.mining, this.nonce, difficulty);
if (hash.substring(0, difficulty) === '0'.repeat(difficulty)) {
//success
this.p2pServer.newBlock(new Block(timestamp, this.lastBlock.hash, hash, this.mining, this.nonce, difficulty));
this.state = Miner.STATE_RESTARTING;
setImmediate(() => { this.startMine() });
} else {
//failure
this.nonce++;
setImmediate(() => { this.mine() });
}
}
}

View file

@ -1,9 +1,12 @@
const Websocket = require('ws');
const N3 = require('n3');
const parser = new N3.Parser(); //({format: 'application/n-quads'});
const DataFactory = require('n3').DataFactory;
const fs = require('fs');
const process = require('process');
const Miner = require('./miner');
const Transaction = require('../wallet/transaction');
const TransactionPool = require('../wallet/transaction-pool');
const Metadata = require('../wallet/metadata');
const P2P_PORT = process.env.P2P_PORT || 5000;
const peers = process.env.PEERS ? process.env.PEERS.split(',') : [];
@ -15,12 +18,13 @@ const MESSAGE_TYPES = {
};
class P2pServer {
constructor(blockchain, transactionPool,chainStorageLocation) {
constructor(blockchain, transactionPool, wallet, chainStorageLocation) {
this.blockchain = blockchain;
this.transactionPool = transactionPool;
this.sockets = [];
this.store = new N3.Store();
this.chainStorageLocation = chainStorageLocation;
this.miner = new Miner(this.blockchain, this.transactionPool, wallet, this);
//possible race if deleted after check, but we live with it I guess
if (fs.existsSync(this.chainStorageLocation)) {
@ -63,39 +67,93 @@ class P2pServer {
const data = JSON.parse(message);
switch(data.type) {
case MESSAGE_TYPES.chain:
newChain(data.chain);
this.newChain(data.chain);
break;
case MESSAGE_TYPES.transaction:
this.transactionPool.updateOrAddTransaction(data.transaction);
this.newTransaction(data.transaction, false);
break;
case MESSAGE_TYPES.metadata:
this.transactionPool.updateOrAddMetadata(data.metadata);
break;
case MESSAGE_TYPES.clear_transactions:
this.transactionPool.clear();
this.newMetadata(data.metadata, false);
break;
//case MESSAGE_TYPES.clear_transactions:
// this.transactionPool.clear();
// break;
}
});
}
newBlock(block) {
this.onNewBlock(block.data);
this.syncChains();
this.persistChain(this.blockchain.chain);
newMetadata(metadata, broadcast) {
if (!Metadata.verifyMetadata(metadata)) {
console.log("Couldn't add metadata to p2pServer, couldn't verify");
return;
}
newChain(chain,persist) {
if (!this.blockchain.replaceChain(chain)) {
switch (this.transactionPool.updateOrAddMetadata(metadata)) {
case TransactionPool.Return.add:
this.miner.startMine();
break;
case TransactionPool.Return.update:
this.miner.interruptIfContainsMetadata(metadata);
break;
case TransactionPool.Return.error:
console.log("Couldn't add metadata to p2pServer, couldn't updateOrAdd");
return;
}
if (broadcast === undefined || broadcast) {
this.broadcastMetadata(metadata);
}
}
newTransaction(transaction, broadcast) {
if (!Transaction.verifyTransaction(transaction)) {
console.log("Couldn't add transaction to p2pServer, couldn't verify");
return false;
}
switch (this.transactionPool.updateOrAddTransaction(transaction)) {
case TransactionPool.Return.add:
this.miner.startMine();
break;
case TransactionPool.Return.update:
this.miner.interruptIfContainsTransaction(transaction);
break;
case TransactionPool.Return.error:
console.log("Couldn't add transaction to p2pServer, couldn't updateOrAdd");
return;
}
if (broadcast === undefined || broadcast) {
this.broadcastTransaction(transaction);
}
}
newBlock(block) {
if (!this.blockchain.addBlock(block)) {
//invalid block, return
return;
}
this.onNewBlock(block);
this.persistChain(this.blockchain.chain);
this.syncChains();
}
newChain(chain, persist) {
const oldChain = this.blockchain.chain;
const divergence = this.blockchain.replaceChain(chain);
if (divergence === null) {
//failed to replace
return;
}
if (typeof persist === "undefined" || persist) {
this.persistChain(chain);
}
//dirty clear
this.store = new N3.Store();
for (var block in this.blockchain.chain) {
this.onNewBlock(block);
for (let i = divergence; i < oldChain.length; i++) {
this.store.deleteGraph(oldChain[i].hash);
}
for (let i = divergence; i < this.blockchain.chain.length; i++) {
this.onNewBlock(this.blockchain.chain[i]);
}
}
@ -112,28 +170,36 @@ class P2pServer {
onNewBlock(block) {
//block data is of form [transactions,metadatas]
if (block.length != 2) {
if (block.data.length != 2) {
//assert?
return;
}
const metadatas = block[1];
for (var metadata in metadatas) {
if (!(SSNmetadata in metadata)) {
this.transactionPool.clearFromBlock(block);
this.miner.interrupt();
const metadatas = block.data[1];
for (const metadata of metadatas) {
if (!("SSNmetadata" in metadata)) {
//assert?
return;
}
var ssn = metadata.SSNmetadata;
const parser = new N3.Parser();
parser.parse(
ssn,
(error, quadN, prefixes) => {
if (quadN) {
store.addQuad(DataFactory.quad(
this.store.addQuad(DataFactory.quad(
DataFactory.namedNode(quadN.subject.id),
DataFactory.namedNode(quadN.predicate.id),
DataFactory.namedNode(quadN.object.id)));
DataFactory.namedNode(quadN.object.id),
DataFactory.namedNode(block.hash)));
}
});
}
@ -163,13 +229,13 @@ class P2pServer {
this.sockets.forEach(socket => this.sendChain(socket));
}
//broadcastTransaction(transaction) {
// this.sockets.forEach(socket => this.sendTransaction(socket, transaction));
//}
broadcastTransaction(transaction) {
this.sockets.forEach(socket => this.sendTransaction(socket, transaction));
}
//broadcastMetadata(metadata) {
// this.sockets.forEach(socket => this.sendMetadata(socket, metadata));
//}
broadcastMetadata(metadata) {
this.sockets.forEach(socket => this.sendMetadata(socket, metadata));
}
//broadcastClearTransactions() {
// this.sockets.forEach(socket => socket.send(JSON.stringify({

View file

@ -8,7 +8,11 @@ class Block {
this.hash = hash;
this.data = data;
this.nonce = nonce;
this.difficulty = difficulty || DIFFICULTY;
if (difficulty === undefined) {
this.difficulty = DIFFICULTY;
} else {
this.difficulty = difficulty;
}
}
toString() {
@ -24,26 +28,20 @@ class Block {
static genesis() {
return new this('Genesis time', '-----', 'f1r57-h45h', [], 0, DIFFICULTY);
}
//we want this to eventually be continously running where there are things in the pool,
//however as node is single threaded, this almost has to be a fiber, and yield after every
//other iteration to allow for meaningful forward progress
//we can either add all new transactions into the block as we see them, or stay with the starting list, idk which
//to be done later
static mineBlock(lastBlock, data) {
let hash, timestamp;
const lastHash = lastBlock.hash;
let { difficulty } = lastBlock;
let nonce = 0;
//returns false if hash doesn't match
static checkHash(hash, timestamp, lastHash, data, nonce, difficulty) {
const computedHash = Block.hash(timestamp, lastHash, data, nonce, difficulty);
do {
nonce++;
timestamp = Date.now();
difficulty = Block.adjustDifficulty(lastBlock, timestamp);
hash = Block.hash(timestamp, lastHash, data, nonce, difficulty);
} while (hash.substring(0, difficulty) !== '0'.repeat(difficulty));
if (computedHash !== hash) {
return false;
}
return new this(timestamp, lastHash, hash, data, nonce, difficulty);
if (hash.substring(0, difficulty) !== '0'.repeat(difficulty)) {
return false;
}
return true;
}
static hash(timestamp, lastHash, data, nonce, difficulty) {
@ -55,11 +53,16 @@ class Block {
return Block.hash(timestamp, lastHash, data, nonce, difficulty);
}
//returns false if block's hash doesn't match internals
static checkBlock(block) {
return Block.checkHash(block.hash, block.timestamp, block.lastHash, block.data, block.nonce, block.difficulty);
}
static adjustDifficulty(lastBlock, currentTime) {
let { difficulty } = lastBlock;
difficulty = lastBlock.timestamp + MINE_RATE > currentTime ?
difficulty + 1 : difficulty - 1;
return difficulty;
return Math.max(0, difficulty);
}
}

View file

@ -5,15 +5,37 @@ class Blockchain {
this.chain = [Block.genesis()];
}
addBlock(data) {
const block = Block.mineBlock(this.chain[this.chain.length-1], data);
this.chain.push(block);
//adds an existing block to the blockchain, returns false if the block can't be added, true if it was added
addBlock(newBlock) {
if (newBlock.lastHash !== this.chain[this.chain.length - 1].hash) {
console.log("Tried to add invalid block, last hash didn't match our last hash");
return false;
}
//how to check if new block's timestamp is believable
if (newBlock.difficulty !== Block.adjustDifficulty(this.chain[this.chain.length - 1], newBlock.timestamp)) {
console.log("Tried to add invalid block, difficulty is incorrect");
return false;
}
if (!Block.checkBlock(newBlock)) {
console.log("Tried to add invalid block, block's hash doesn't match its contents");
return false;
}
return block;
this.chain.push(newBlock);
console.log("Added new block: ");
//console.log(newBlock);
return true;
}
isValidChain(chain) {
if(JSON.stringify(chain[0]) !== JSON.stringify(Block.genesis())) return false;
if (chain.length === 0) {
return false;
}
if (JSON.stringify(chain[0]) !== JSON.stringify(Block.genesis())) {
return false;
}
for (let i=1; i<chain.length; i++) {
const block = chain[i];
@ -23,12 +45,15 @@ class Blockchain {
block.hash !== Block.blockHash(block)) {
return false;
}
if (!Block.checkBlock(block)) {
return false;
}
}
return true;
}
//return false on failure, true on success
//return null on fail, returns the index of where they differ
replaceChain(newChain) {
if (newChain.length <= this.chain.length) {
console.log('Received chain is not longer than the current chain.');
@ -39,8 +64,18 @@ class Blockchain {
}
console.log('Replacing blockchain with the new chain.');
const oldChain = this.chain;
this.chain = newChain;
return true;
//find where they differ
for (let i = 1; i < oldChain.length; ++i) {
if (oldChain[i].hash !== newChain[i].hash) {
return i;
}
}
//if they didn't differ in the length of the old chain, must be one after
return oldChain.length;
}
}

View file

@ -1,12 +1,12 @@
const ChainUtil = require('../chain-util');
const Transaction = require('./transaction');
const { INITIAL_BALANCE } = require('../config');
const Metadata = require('./metadata');
const ChainUtil = require('../chain-util');
class Wallet {
constructor() {
constructor(keyPair) {
this.balance = INITIAL_BALANCE;
this.keyPair = ChainUtil.genKeyPair();
this.keyPair = keyPair;
this.publicKey = this.keyPair.getPublic().encode('hex');
}
@ -25,36 +25,16 @@ class Wallet {
if (amount > this.balance) {
console.log(`Amount: ${amount} exceceds current balance: ${this.balance}`);
return;
return null;
}
let transaction = transactionPool.existingTransaction(this.publicKey);
if (transaction) {
transaction.update(this, recipient, amount);
} else {
transaction = Transaction.newTransaction(this, recipient, amount);
transactionPool.updateOrAddTransaction(transaction);
return Transaction.newTransaction(this, recipient, amount);
}
return transaction;
createMetadata(SSNmetadata) {
return Metadata.newMetadata(this, SSNmetadata);
}
createMetadata(SSNmetadata, transactionPool){
//let metadata = transactionPool.existingMetadata(this.publicKey);
// if (metaData) {
// metadata.update(this, Geo, Std, Name,MetaHash,file);
// } else {*/
let metadata= Metadata.newMetadata(this, SSNmetadata);
transactionPool.AddMetadata(metadata);
//}
return metadata;
}
calculateBalance(blockchain) {
let balance = this.balance;
let transactions = [];
@ -93,7 +73,7 @@ class Wallet {
}
static blockchainWallet() {
const blockchainWallet = new this();
const blockchainWallet = new this(ChainUtil.genKeyPair());
blockchainWallet.address = 'blockchain-wallet';
return blockchainWallet;
}

View file

@ -1,29 +1,50 @@
const Transaction = require('../wallet/transaction');
const Metadata = require('../wallet/metadata')
const Return = {
add: 1,
update: 2,
error: 3
};
class TransactionPool {
constructor() {
this.transactions = [];
this.metadataS =[];
}
//returns true on update, false on add
updateOrAddTransaction(transaction) {
let transactionWithId = this.transactions.find(t => t.id === transaction.id);
if (!Transaction.verifyTransaction(transaction)) {
console.log("Couldn't update or add transaction, transaction couldn't be verified");
return Return.error;
}
const foundIndex = this.transactions.findIndex(t => t.id === transaction.id);
if (transactionWithId) {
this.transactions[this.transactions.indexOf(transactionWithId)] = transaction;
if (foundIndex !== -1) {
this.transactions[foundIndex] = transaction;
return Return.update;
} else {
this.transactions.push(transaction);
return Return.add;
}
}
AddMetadata(metadata) {
// let metadataWithId = this.metadataS.find(t => t.id === metadata.id);
updateOrAddMetadata(metadata) {
if (!Metadata.verifyMetadata(metadata)) {
console.log("Couldn't update metdata, metadata couldn't be verified");
return Return.error;
}
// if (metadataWithId) {
// this.metaDataS[this.metadataS.indexOf(metadataWithId)] = metadata;
// } else {
const foundIndex = this.metadataS.findIndex(t => t.id === metadata.id);
if (foundIndex !== -1) {
this.metadataS[foundIndex] = metadata;
return Return.update;
} else {
this.metadataS.push(metadata);
// }
return Return.add;
}
}
existingTransaction(address) {
@ -64,10 +85,31 @@ class TransactionPool {
});
}
clear() {
clearFromBlock(block) {
const transactions = block.data[0];
const metadatas = block.data[1];
for (const transaction of transactions) {
const foundTransaction = this.transactions.findIndex(t => t.id === transaction.id);
if (foundTransaction !== -1) {
this.transactions.splice(foundTransaction, 1);
}
}
for (const metadata of metadatas) {
const foundMetadata = this.metadataS.findIndex(m => m.id === metadata.id);
if (foundMetadata !== -1) {
this.metadataS.splice(foundMetadata, 1);
}
}
}
clearAll() {
this.transactions = [];
this.metadataS = [];
}
}
module.exports = TransactionPool;
module.exports.Return = Return;