From 23b46a3dd5d09f25747052e7617c26507a9e28f5 Mon Sep 17 00:00:00 2001 From: mjezek Date: Sun, 13 Mar 2022 12:47:45 +0100 Subject: [PATCH 1/3] Delete old unnecessary ping records, store player count records in separate table --- config.json | 1 + lib/app.js | 4 +++ lib/database.js | 91 ++++++++++++++++++++++++++++++++++++++++++++++++- lib/servers.js | 3 ++ 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/config.json b/config.json index 327efe7..462577c 100644 --- a/config.json +++ b/config.json @@ -9,6 +9,7 @@ }, "logFailedPings": true, "logToDatabase": false, + "deleteOldPings": false, "graphDuration": 86400000, "serverGraphDuration": 180000 } diff --git a/lib/app.js b/lib/app.js index 2e1daeb..2048d90 100644 --- a/lib/app.js +++ b/lib/app.js @@ -22,6 +22,10 @@ class App { // Setup database instance this.database.ensureIndexes(() => { this.database.loadGraphPoints(config.graphDuration, () => { + if (config.deleteOldPings) { + this.database.initOldPingsDeleter() + } + this.database.loadRecords(callback) }) }) diff --git a/lib/database.js b/lib/database.js index 48b74d2..201384f 100644 --- a/lib/database.js +++ b/lib/database.js @@ -52,6 +52,7 @@ class Database { this._sql.serialize(() => { this._sql.run('CREATE TABLE IF NOT EXISTS pings (timestamp BIGINT NOT NULL, ip TINYTEXT, playerCount MEDIUMINT)', handleError) + this._sql.run('CREATE TABLE IF NOT EXISTS players_record (timestamp BIGINT, ip TINYTEXT, playerCount MEDIUMINT)', handleError) this._sql.run('CREATE INDEX IF NOT EXISTS ip_index ON pings (ip, playerCount)', handleError) this._sql.run('CREATE INDEX IF NOT EXISTS timestamp_index on PINGS (timestamp)', [], err => { handleError(err) @@ -130,6 +131,34 @@ class Database { playerCount, timestamp: TimeTracker.toSeconds(timestamp) } + } else { + this.getRecordLegacy(serverRegistration.data.ip, (hasRecordLegacy, playerCountLegacy, timestampLegacy) => { + // New values that will be inserted to table + let newTimestamp = null + let newPlayerCount = null + + // If legacy record found, use it for insertion + if (hasRecordLegacy) { + newTimestamp = timestampLegacy + newPlayerCount = playerCountLegacy + } + + // Set record to recordData + serverRegistration.recordData = { + playerCount: newPlayerCount, + timestamp: TimeTracker.toSeconds(newTimestamp) + } + + // Insert server entry to records table + const statement = this._sql.prepare('INSERT INTO players_record (timestamp, ip, playerCount) VALUES (?, ?, ?)') + statement.run(newTimestamp, serverRegistration.data.ip, newPlayerCount, err => { + if (err) { + logger.error(`Cannot insert initial player count record of ${serverRegistration.data.ip}`) + throw err + } + }) + statement.finalize() + }) } // Check if completedTasks hit the finish value @@ -155,7 +184,7 @@ class Database { } getRecord (ip, callback) { - this._sql.all('SELECT MAX(playerCount), timestamp FROM pings WHERE ip = ?', [ + this._sql.all('SELECT playerCount, timestamp FROM players_record WHERE ip = ?', [ ip ], (err, data) => { if (err) { @@ -163,6 +192,32 @@ class Database { throw err } + // Record not found + if (data[0] === undefined) { + // eslint-disable-next-line node/no-callback-literal + callback(false) + return + } + + const playerCount = data[0].playerCount + const timestamp = data[0].timestamp + + // Allow null player counts and timestamps, the frontend will safely handle them + // eslint-disable-next-line node/no-callback-literal + callback(true, playerCount, timestamp) + }) + } + + // Retrieves record from pings table, used for converting to separate table + getRecordLegacy (ip, callback) { + this._sql.all('SELECT MAX(playerCount), timestamp FROM pings WHERE ip = ?', [ + ip + ], (err, data) => { + if (err) { + logger.log('error', `Cannot get legacy ping record for ${ip}`) + throw err + } + // For empty results, data will be length 1 with [null, null] const playerCount = data[0]['MAX(playerCount)'] const timestamp = data[0].timestamp @@ -200,6 +255,40 @@ class Database { }) statement.finalize() } + + updatePlayerCountRecord (ip, playerCount, timestamp) { + const statement = this._sql.prepare('UPDATE players_record SET timestamp = ?, playerCount = ? WHERE ip = ?') + statement.run(timestamp, playerCount, ip, err => { + if (err) { + logger.error(`Cannot update player count record of ${ip} at ${timestamp}`) + throw err + } + }) + statement.finalize() + } + + initOldPingsDeleter () { + // Delete old records every hour + setInterval(() => this.deleteOldPingRecords(), 3600000) + } + + deleteOldPingRecords () { + // The oldest timestamp that will be kept + const oldestTimestamp = TimeTracker.getEpochMillis() - config.graphDuration + + const deleteStart = TimeTracker.getEpochMillis() + const statement = this._sql.prepare('DELETE FROM pings WHERE timestamp < ?;') + statement.run(oldestTimestamp, err => { + if (err) { + logger.error('Cannot delete old ping records') + throw err + } else { + const deleteTook = TimeTracker.getEpochMillis() - deleteStart + logger.info(`Old ping records deleted in ${deleteTook}ms`) + } + }) + statement.finalize() + } } module.exports = Database diff --git a/lib/servers.js b/lib/servers.js index 944cdb4..4dc48e0 100644 --- a/lib/servers.js +++ b/lib/servers.js @@ -63,6 +63,9 @@ class ServerRegistration { // Append an updated recordData update.recordData = this.recordData + + // Update record in database + this._app.database.updatePlayerCountRecord(this.data.ip, resp.players.online, timestamp) } if (this.updateFavicon(resp.favicon)) { From 37a1579b371e87452807d147c1129cb09e99f080 Mon Sep 17 00:00:00 2001 From: mjezek Date: Sun, 22 May 2022 16:28:58 +0200 Subject: [PATCH 2/3] Delete old pings also on startup --- lib/app.js | 12 +++++++----- lib/database.js | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/app.js b/lib/app.js index 2048d90..ddc4408 100644 --- a/lib/app.js +++ b/lib/app.js @@ -22,11 +22,13 @@ class App { // Setup database instance this.database.ensureIndexes(() => { this.database.loadGraphPoints(config.graphDuration, () => { - if (config.deleteOldPings) { - this.database.initOldPingsDeleter() - } - - this.database.loadRecords(callback) + this.database.loadRecords(() => { + if (config.deleteOldPings) { + this.database.initOldPingsDelete(callback) + } else { + callback() + } + }) }) }) } diff --git a/lib/database.js b/lib/database.js index 201384f..9b4828b 100644 --- a/lib/database.js +++ b/lib/database.js @@ -267,12 +267,18 @@ class Database { statement.finalize() } - initOldPingsDeleter () { - // Delete old records every hour - setInterval(() => this.deleteOldPingRecords(), 3600000) + initOldPingsDelete (callback) { + // Delete old ping records on startup + logger.info('Deleting old ping records..') + this.deleteOldPingRecords(() => { + // Delete old ping records every hour + setInterval(() => this.deleteOldPingRecords(), 3600000) + + callback() + }) } - deleteOldPingRecords () { + deleteOldPingRecords (callback) { // The oldest timestamp that will be kept const oldestTimestamp = TimeTracker.getEpochMillis() - config.graphDuration @@ -285,6 +291,10 @@ class Database { } else { const deleteTook = TimeTracker.getEpochMillis() - deleteStart logger.info(`Old ping records deleted in ${deleteTook}ms`) + + if (callback !== undefined) { + callback() + } } }) statement.finalize() From 4f81041e97157e3fff4914dcad63cb6045b129a3 Mon Sep 17 00:00:00 2001 From: mjezek Date: Sat, 29 Oct 2022 18:51:14 +0200 Subject: [PATCH 3/3] Modified old pings cleanup config, ip as primary key in player records table --- config.json | 5 ++++- lib/app.js | 2 +- lib/database.js | 23 +++++++++++++---------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/config.json b/config.json index 462577c..d8d84f9 100644 --- a/config.json +++ b/config.json @@ -7,9 +7,12 @@ "pingAll": 3000, "connectTimeout": 2500 }, + "oldPingsCleanup": { + "enabled": false, + "interval": 3600000 + }, "logFailedPings": true, "logToDatabase": false, - "deleteOldPings": false, "graphDuration": 86400000, "serverGraphDuration": 180000 } diff --git a/lib/app.js b/lib/app.js index ddc4408..965438e 100644 --- a/lib/app.js +++ b/lib/app.js @@ -23,7 +23,7 @@ class App { this.database.ensureIndexes(() => { this.database.loadGraphPoints(config.graphDuration, () => { this.database.loadRecords(() => { - if (config.deleteOldPings) { + if (config.oldPingsCleanup && config.oldPingsCleanup.enabled) { this.database.initOldPingsDelete(callback) } else { callback() diff --git a/lib/database.js b/lib/database.js index 9b4828b..c0d5caa 100644 --- a/lib/database.js +++ b/lib/database.js @@ -52,7 +52,7 @@ class Database { this._sql.serialize(() => { this._sql.run('CREATE TABLE IF NOT EXISTS pings (timestamp BIGINT NOT NULL, ip TINYTEXT, playerCount MEDIUMINT)', handleError) - this._sql.run('CREATE TABLE IF NOT EXISTS players_record (timestamp BIGINT, ip TINYTEXT, playerCount MEDIUMINT)', handleError) + this._sql.run('CREATE TABLE IF NOT EXISTS players_record (timestamp BIGINT, ip TINYTEXT NOT NULL PRIMARY KEY, playerCount MEDIUMINT)', handleError) this._sql.run('CREATE INDEX IF NOT EXISTS ip_index ON pings (ip, playerCount)', handleError) this._sql.run('CREATE INDEX IF NOT EXISTS timestamp_index on PINGS (timestamp)', [], err => { handleError(err) @@ -268,17 +268,20 @@ class Database { } initOldPingsDelete (callback) { - // Delete old ping records on startup - logger.info('Deleting old ping records..') - this.deleteOldPingRecords(() => { - // Delete old ping records every hour - setInterval(() => this.deleteOldPingRecords(), 3600000) + // Delete old pings on startup + logger.info('Deleting old pings..') + this.deleteOldPings(() => { + const oldPingsCleanupInterval = config.oldPingsCleanup.interval || 3600000 + if (oldPingsCleanupInterval > 0) { + // Delete old pings periodically + setInterval(() => this.deleteOldPings(), oldPingsCleanupInterval) + } callback() }) } - deleteOldPingRecords (callback) { + deleteOldPings (callback) { // The oldest timestamp that will be kept const oldestTimestamp = TimeTracker.getEpochMillis() - config.graphDuration @@ -286,13 +289,13 @@ class Database { const statement = this._sql.prepare('DELETE FROM pings WHERE timestamp < ?;') statement.run(oldestTimestamp, err => { if (err) { - logger.error('Cannot delete old ping records') + logger.error('Cannot delete old pings') throw err } else { const deleteTook = TimeTracker.getEpochMillis() - deleteStart - logger.info(`Old ping records deleted in ${deleteTook}ms`) + logger.info(`Old pings deleted in ${deleteTook}ms`) - if (callback !== undefined) { + if (callback) { callback() } }