Update 0.4.0: Major refactor and debug command

README.md

@@ -1,7 +1,27 @@
 # MarkBot for Discord
 This is a super rough prototype. A Markov chain bot using markov-strings. Just uploading here so it can run and generate training data.
 
+# Setup
+
+## Configuration
+Create a file called `config.json` in the project directory with the contents:
+```
+{
+  "prefix":"!mark",
+  "game":"\"!mark help\" for help",
+  "token":"k5NzE2NDg1MTIwMjc0ODQ0Nj.DSnXwg.ttNotARealToken5p3WfDoUxhiH"
+}
+```
+
 ## Changelog
+### 0.4.0
+Huge refactor. 
+Added `!mark debug` which sends debug info alongside the message.
+Converted the fetchMessages function to async/await (updating the requirement to Node.js 8).
+Updated module versions.
+Added faster unique-array-by-property function
+Added linting and linted the project. 
+
 ### 0.3.0
 Added TTS support and random message attachments.
 Deleted messages no longer persist in the database longer than 24 hours.

_start.bat

@@ -1 +0,0 @@
-npm start

_start.sh

@@ -1,2 +0,0 @@
-#!/bin/bash
-npm start

index.js

@@ -1,226 +1,286 @@
-const Discord = require('discord.js') //https://discord.js.org/#/docs/main/stable/general/welcome
-const fs = require('fs')
-const Markov = require('markov-strings')
-const uniqueBy = require('unique-by');
+const Discord = require('discord.js'); // https://discord.js.org/#/docs/main/stable/general/welcome
+const fs = require('fs');
+const Markov = require('markov-strings');
 const schedule = require('node-schedule');
-const client = new Discord.Client()
-const ZEROWIDTH_SPACE = String.fromCharCode(parseInt('200B', 16))
-const MAXMESSAGELENGTH = 2000
 
+const client = new Discord.Client();
+// const ZEROWIDTH_SPACE = String.fromCharCode(parseInt('200B', 16));
+// const MAXMESSAGELENGTH = 2000;
 
-
-let guilds = []
-let connected = -1
-let GAME = 'GAME'
-let BOTDESC = 'Amazing.'
-let PREFIX = '! '
-let VOLUME
-let inviteCmd = 'invite'
-let commands = {}
-let aliases = {}
-let errors = []
+const PAGE_SIZE = 100;
+// let guilds = [];
+// let connected = -1;
+let GAME = 'GAME';
+let PREFIX = '! ';
+const inviteCmd = 'invite';
+const errors = [];
 
 let fileObj = {
-  messages: []
-}
+  messages: [],
+};
 
-let markovDB = []
-let messageCache = []
-let deletionCache = []
+let markovDB = [];
+let messageCache = [];
+let deletionCache = [];
 const markovOpts = {
-  maxLength: 400,
+  stateSize: 2,
+  maxLength: 2000,
   minWords: 3,
-  minScore: 10
-}
-let markov
+  maxWords: 0,
+  minScore: 10,
+  minScorePerWord: 0,
+  maxTries: 10000,
+};
+let markov;
 // let markov = new Markov(markovDB, markovOpts);
 
-function regenMarkov() {
-  console.log("Regenerating Markov corpus...")
-  try {
-    fileObj = JSON.parse(fs.readFileSync('markovDB.json', 'utf8'))
+function uniqueBy(arr, propertyName) {
+  const unique = [];
+  const found = {};
+
+  for (let i = 0; i < arr.length; i++) {
+    const value = arr[i][propertyName];
+    if (!found[value]) {
+      found[value] = true;
+      unique.push(arr[i]);
+    }
+  }
+  return unique;
+}
+
+/**
+ * Regenerates the corpus and saves all cached changes to disk
+ */
+function regenMarkov() {
+  console.log('Regenerating Markov corpus...');
+  try {
+    fileObj = JSON.parse(fs.readFileSync('markovDB.json', 'utf8'));
+  } catch (err) {
+    console.log(err);
   }
-  catch (err) { console.log(err) }
   // console.log("MessageCache", messageCache)
-  markovDB = fileObj.messages
-  markovDB = uniqueBy(markovDB.concat(messageCache), 'id')
-  deletionCache.forEach(id => {
-    let removeIndex = markovDB.map(function (item) { return item.id; }).indexOf(id)
+  markovDB = fileObj.messages;
+  markovDB = uniqueBy(markovDB.concat(messageCache), 'id');
+  deletionCache.forEach((id) => {
+    const removeIndex = markovDB.map(item => item.id).indexOf(id);
     // console.log('Remove Index:', removeIndex)
-    markovDB.splice(removeIndex, 1)
-  })
-  deletionCache = []
-  if (markovDB.length == 0)
-    markovDB.push({ string: 'hello', id: null })
+    markovDB.splice(removeIndex, 1);
+  });
+  deletionCache = [];
+  if (markovDB.length === 0) {
+    markovDB.push({ string: 'hello', id: null });
+  }
   markov = new Markov(markovDB, markovOpts);
-  markov.buildCorpusSync()
-  fileObj.messages = markovDB
+  markov.buildCorpusSync();
+  fileObj.messages = markovDB;
   // console.log("WRITING THE FOLLOWING DATA:")
   // console.log(fileObj)
-  fs.writeFileSync('markovDB.json', JSON.stringify(fileObj), 'utf-8')
+  fs.writeFileSync('markovDB.json', JSON.stringify(fileObj), 'utf-8');
   fileObj = null;
-  // markovDB = []
-  messageCache = []
-  console.log("Done regenerating Markov corpus.")
+  messageCache = [];
+  console.log('Done regenerating Markov corpus.');
 }
 
+/**
+ * Loads the config settings from disk
+ */
 function loadConfig() {
-  let cfgfile = 'config.json'
+  const cfgfile = 'config.json';
   if (fs.existsSync(cfgfile)) {
-    let cfg = JSON.parse(fs.readFileSync(cfgfile, 'utf8'))
-    PREFIX = cfg.prefix
-    GAME = cfg.game
-    BOTDESC = cfg.description
-    inviteCmd = cfg.invitecmd
-    //regenMarkov()
-    client.login(cfg.token)
-  }
-  else {
-    console.log('Oh no!!! ' + cfgfile + ' could not be found!')
+    const cfg = JSON.parse(fs.readFileSync(cfgfile, 'utf8'));
+    PREFIX = cfg.prefix;
+    GAME = cfg.game;
+    // regenMarkov()
+    client.login(cfg.token);
+  } else {
+    console.log(`Oh no!!! ${cfgfile} could not be found!`);
   }
 }
 
-client.on('ready', () => {
-  console.log('Markbot by Charlie Laabs')
-  client.user.setActivity(GAME)
-})
-
-client.on('error', (err) => {
-  let errText = 'ERROR: ' + err.name + ' - ' + err.message
-  console.log(errText)
-  errors.push(errText)
-  fs.writeFile('error.json', JSON.stringify(errors), function (err) {
-    if (err)
-      console.log('error writing to error file: ' + err.message)
-  })
-})
-
-client.on('message', message => {
-  if (message.guild) {
-    let command = validateMessage(message)
-    if (command === 'help') {
-      let richem = new Discord.RichEmbed()
-        .setAuthor(client.user.username, client.user.avatarURL)
-        .setThumbnail(client.user.avatarURL)
-        .setDescription('A Markov chain chatbot that speaks based on previous chat input.')
-        .addField('!mark', 'Generates a sentence to say based on the chat database. Send your message as TTS to recieve it as TTS.')
-        .addField('!mark train', 'Fetches the maximum amount of previous messages in the current text channel, adds it to the database, and regenerates the corpus. Takes about 2 minutes.')
-        .addField('!mark regen', 'Manually regenerates the corpus to add recent chat info. Run this before shutting down to avoid any data loss. This automatically runs at midnight.')
-        .addField('!mark invite', 'Don\'t invite this bot to other servers. The database is shared between all servers and text channels.')
-      message.channel.send(richem)
-        .catch(reason => {
-          message.author.send(richem)
-        })
-    }
-    if (command === 'train') {
-      console.log("Training...")
-      fileObj = {
-        messages: []
-      }
-      fs.writeFileSync('markovDB.json', JSON.stringify(fileObj), 'utf-8')
-      fetchMessageChunk(message, null, [])
-    }
-    if (command === 'respond') {
-      console.log("Responding...")
-      markov.generateSentence().then(result => {
-        console.log('Generated Result:', result)
-        let messageOpts = {
-          tts: message.tts
-        }
-        let randomMessage = markovDB[Math.floor(Math.random() * markovDB.length)]
-        console.log('Random Message:', randomMessage)
-        if (randomMessage.hasOwnProperty('attachment')) {
-          messageOpts.files = [{
-            attachment: randomMessage.attachment
-          }]
-        }
-        message.channel.send(result.string, messageOpts)
-      }).catch(err => {
-        console.log(err)
-        if (err.message == 'Cannot build sentence with current corpus and options')
-          // message.reply('Not enough chat data for a response. Run `!mark train` to process past messages.')
-          console.log('Not enough chat data for a response.')
-      })
-    }
-    if (command === 'regen') {
-      console.log("Regenerating...")
-      regenMarkov()
-    }
-    if (command === null) {
-      console.log("Listening...")
-      if (!message.author.bot) {
-        let dbObj = {
-          string: message.content,
-          id: message.id
-        }
-        if (message.attachments.size > 0) {
-          dbObj.attachment = message.attachments.values().next().value.url
-        }
-        messageCache.push(dbObj)
-      }
-    }
-    if (command === inviteCmd) {
-      let richem = new Discord.RichEmbed()
-        .setAuthor('Invite ' + client.user.username, client.user.avatarURL)
-        .setThumbnail(client.user.avatarURL)
-        .addField('Invite', "[Invite " + client.user.username + " to your server](https://discordapp.com/oauth2/authorize?client_id=" + client.user.id + "&scope=bot)")
-
-      message.channel.send(richem)
-        .catch(reason => {
-          message.author.send(richem)
-        })
-    }
-  }
-})
-
-client.on('messageDelete', message => {
-  // console.log('Adding message ' + message.id + ' to deletion cache.')
-  deletionCache.push(message.id)
-  console.log('deletionCache:', deletionCache)
-})
-
+/**
+ * Reads a new message and checks if and which command it is.
+ * @param {Message} message Message to be interpreted as a command
+ * @return {String} Command string
+ */
 function validateMessage(message) {
-  let messageText = message.content.toLowerCase()
+  const messageText = message.content.toLowerCase();
   let command = null;
-  let thisPrefix = messageText.substring(0, PREFIX.length)
-  if (thisPrefix == PREFIX) {
-    let split = messageText.split(" ")
-    if (split[0] == PREFIX && split.length == 1)
-      command = 'respond'
-    else if (split[1] == 'train')
-      command = 'train'
-    else if (split[1] == 'help')
-      command = 'help'
-    else if (split[1] == 'regen')
-      command = 'regen'
-    else if (split[1] == 'invite')
-      command = 'invite'
+  const thisPrefix = messageText.substring(0, PREFIX.length);
+  if (thisPrefix === PREFIX) {
+    const split = messageText.split(' ');
+    if (split[0] === PREFIX && split.length === 1) {
+      command = 'respond';
+    } else if (split[1] === 'train') {
+      command = 'train';
+    } else if (split[1] === 'help') {
+      command = 'help';
+    } else if (split[1] === 'regen') {
+      command = 'regen';
+    } else if (split[1] === 'invite') {
+      command = 'invite';
+    } else if (split[1] === 'debug') {
+      command = 'debug';
     }
-  return command
+  }
+  return command;
 }
 
-function fetchMessageChunk(message, oldestMessageID, historyCache) {
-  message.channel.fetchMessages({ before: oldestMessageID, limit: 100 })
-    .then(messages => {
-      historyCache = historyCache.concat(messages.filter(elem => !elem.author.bot).map(elem => {
-        let dbObj = {
+/**
+ * Function to recursively get all messages in a text channel's history. Ends
+ * by regnerating the corpus.
+ * @param {Message} message Message initiating the command, used for getting
+ * channel data
+ */
+async function fetchMessages(message) {
+  let historyCache = [];
+  let keepGoing = true;
+  let oldestMessageID = null;
+
+  while (keepGoing) {
+    // eslint-disable-next-line no-await-in-loop
+    const messages = await message.channel.fetchMessages({
+      before: oldestMessageID,
+      limit: PAGE_SIZE,
+    });
+    const nonBotMessageFormatted = messages
+      .filter(elem => !elem.author.bot).map((elem) => {
+        const dbObj = {
           string: elem.content,
-          id: elem.id
-        }
+          id: elem.id,
+        };
         if (elem.attachments.size > 0) {
-          dbObj.attachment = elem.attachments.values().next().value.url
+          dbObj.attachment = elem.attachments.values().next().value.url;
+        }
+        return dbObj;
+      });
+    historyCache = historyCache.concat(nonBotMessageFormatted);
+    oldestMessageID = messages.last().id;
+    if (messages.size < PAGE_SIZE) {
+      keepGoing = false;
+    }
+  }
+  console.log(`Trained from ${historyCache.length} past human authored messages.`);
+  messageCache = messageCache.concat(historyCache);
+  regenMarkov();
+  message.reply(`Finished training from past ${historyCache.length} messages.`);
+}
+
+
+/**
+ * General Markov-chain response function
+ * @param {Message} message The message that invoked the action, used for channel info.
+ * @param {Boolean} debug Sends debug info as a message if true.
+ */
+function generateResponse(message, debug = false) {
+  console.log('Responding...');
+  markov.generateSentence().then((result) => {
+    console.log('Generated Result:', result);
+    const messageOpts = { tts: message.tts };
+    const randomMessage = markovDB[Math.floor(Math.random() * markovDB.length)];
+    console.log('Random Message:', randomMessage);
+    if (Object.prototype.hasOwnProperty.call(randomMessage, 'attachment')) {
+      messageOpts.files = [{ attachment: randomMessage.attachment }];
+    }
+    message.channel.send(result.string, messageOpts);
+    if (debug) message.channel.send(`\`\`\`\n${JSON.stringify(result, null, 2)}\n\`\`\``);
+  }).catch((err) => {
+    console.log(err);
+    if (debug) message.channel.send(`\n\`\`\`\nERROR${err}\n\`\`\``);
+    if (err.message.includes('Cannot build sentence with current corpus')) {
+      console.log('Not enough chat data for a response.');
     }
-        return dbObj
-      }));
-      oldestMessageID = messages.last().id
-      return historyCache.concat(fetchMessageChunk(message, oldestMessageID, historyCache))
-    }).catch(err => {
-      console.log("Trained from " + historyCache.length + " past messages.")
-      messageCache = messageCache.concat(historyCache)
-      regenMarkov()
-      message.reply('Finished training from past ' + historyCache.length + ' messages.')
   });
 }
 
-loadConfig()
-const daily = schedule.scheduleJob('0 0 * * *', regenMarkov());
+
+client.on('ready', () => {
+  console.log('Markbot by Charlie Laabs');
+  client.user.setActivity(GAME);
+});
+
+client.on('error', (err) => {
+  const errText = `ERROR: ${err.name} - ${err.message}`;
+  console.log(errText);
+  errors.push(errText);
+  fs.writeFile('error.json', JSON.stringify(errors), (fsErr) => {
+    if (fsErr) {
+      console.log(`error writing to error file: ${fsErr.message}`);
+    }
+  });
+});
+
+client.on('message', (message) => {
+  if (message.guild) {
+    const command = validateMessage(message);
+    if (command === 'help') {
+      console.log(message.channel);
+      const richem = new Discord.RichEmbed()
+        .setAuthor(client.user.username, client.user.avatarURL)
+        .setThumbnail(client.user.avatarURL)
+        .setDescription('A Markov chain chatbot that speaks based on previous chat input.')
+        .addField('!mark', 'Generates a sentence to say based on the chat database. Send your '
+        + 'message as TTS to recieve it as TTS.')
+        .addField('!mark train', 'Fetches the maximum amount of previous messages in the current '
+        + 'text channel, adds it to the database, and regenerates the corpus. Takes some time.')
+        .addField('!mark regen', 'Manually regenerates the corpus to add recent chat info. Run '
+        + 'this before shutting down to avoid any data loss. This automatically runs at midnight.')
+        .addField('!mark invite', 'Don\'t invite this bot to other servers. The database is shared '
+        + 'between all servers and text channels.')
+        .addBlankField('!mark debug', 'Runs the !mark command and follows it up with debug info.');
+      message.channel.send(richem).catch(() => {
+        message.author.send(richem);
+      });
+    }
+    if (command === 'train') {
+      console.log('Training...');
+      fileObj = {
+        messages: [],
+      };
+      fs.writeFileSync('markovDB.json', JSON.stringify(fileObj), 'utf-8');
+      fetchMessages(message);
+    }
+    if (command === 'respond') {
+      generateResponse(message);
+    }
+    if (command === 'debug') {
+      generateResponse(message, true);
+    }
+    if (command === 'regen') {
+      console.log('Regenerating...');
+      regenMarkov();
+    }
+    if (command === null) {
+      console.log('Listening...');
+      if (!message.author.bot) {
+        const dbObj = {
+          string: message.content,
+          id: message.id,
+        };
+        if (message.attachments.size > 0) {
+          dbObj.attachment = message.attachments.values().next().value.url;
+        }
+        messageCache.push(dbObj);
+      }
+    }
+    if (command === inviteCmd) {
+      const richem = new Discord.RichEmbed()
+        .setAuthor(`Invite ${client.user.username}`, client.user.avatarURL)
+        .setThumbnail(client.user.avatarURL)
+        .addField('Invite', `[Invite ${client.user.username} to your server](https://discordapp.com/oauth2/authorize?client_id=${client.user.id}&scope=bot)`);
+
+      message.channel.send(richem)
+        .catch(() => {
+          message.author.send(richem);
+        });
+    }
+  }
+});
+
+client.on('messageDelete', (message) => {
+  // console.log('Adding message ' + message.id + ' to deletion cache.')
+  deletionCache.push(message.id);
+  console.log('deletionCache:', deletionCache);
+});
+
+loadConfig();
+schedule.scheduleJob('0 0 * * *', regenMarkov());

package.json

@@ -1,6 +1,6 @@
 {
   "name": "markbot",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "A conversational Markov chain bot for Discord",
   "main": "index.js",
   "scripts": {
@@ -16,11 +16,39 @@
   "author": "Charlie Laabs <charlielaabs@gmail.com>",
   "license": "MIT",
   "dependencies": {
-    "discord.js": "^11.3.2",
+    "bufferutil": "^4.0.0",
+    "discord.js": "^11.4.2",
     "erlpack": "github:discordapp/erlpack",
-    "markov-strings": "^1.3.5",
+    "markov-strings": "^1.5.0",
     "node-schedule": "^1.3.0",
-    "unique-by": "^1.0.0",
     "zlib-sync": "^0.1.4"
+  },
+  "engines": {
+    "node": ">=8.0.0"
+  },
+  "devDependencies": {
+    "eslint": "^5.4.0",
+    "eslint-config-airbnb-base": "^13.1.0",
+    "eslint-plugin-import": "^2.14.0"
+  },
+  "eslintConfig": {
+    "parserOptions": {
+      "ecmaVersion": 2017
+    },
+    "env": {
+      "node": true
+    },
+    "extends": [
+      "airbnb-base"
+    ],
+    "rules": {
+      "no-console": 0,
+      "no-plusplus": [
+        "error",
+        {
+          "allowForLoopAfterthoughts": true
+        }
+      ]
+    }
   }
 }