pacnpal/markov-discord
1
0
Fork 0
mirror of https://github.com/pacnpal/markov-discord.git synced 2025-12-20 03:01:04 -05:00
Code Issues Packages Projects Releases Wiki Activity

Initial rough working draft of v3

Browse Source
This commit is contained in:
Charlie Laabs
2021-12-19 16:59:13 -06:00
parent b777604bb3
commit 466c122dd2
27 changed files with 7581 additions and 1663 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.eslintrc.js
View File

@@ -6,7 +6,6 @@ module.exports = {
extends: [ extends: [
'airbnb-base', 'airbnb-base',
'plugin:@typescript-eslint/recommended', 'plugin:@typescript-eslint/recommended',
'prettier/@typescript-eslint',
'plugin:prettier/recommended', 'plugin:prettier/recommended',
], ],
parser: '@typescript-eslint/parser', parser: '@typescript-eslint/parser',
@@ -15,6 +14,12 @@ module.exports = {
sourceType: 'module' sourceType: 'module'
}, },
plugins: ['@typescript-eslint'], plugins: ['@typescript-eslint'],
rules: {
'import/extensions': 0,
'import/prefer-default-export': 0,
'no-shadow': 'off',
'@typescript-eslint/no-shadow': ['error'],
},
settings: { settings: {
'import/extensions': ['.js', '.ts',], 'import/extensions': ['.js', '.ts',],
'import/parsers': { 'import/parsers': {
@@ -25,5 +30,6 @@ module.exports = {
extensions: ['.js', '.ts',] extensions: ['.js', '.ts',]
} }
} }
} },
ignorePatterns: ['dist/**', 'node_modules/**', '.eslintrc.js']
} }

1
.nvmrc Normal file
View File

@@ -0,0 +1 @@
16

1
.prettierrc.js
View File

@@ -1,5 +1,4 @@
module.exports = { module.exports = {
printWidth: 100, printWidth: 100,
singleQuote: true, singleQuote: true,
trailingComma: 'es5'
} }

2
.vscode/launch.json vendored
View File

@@ -13,7 +13,7 @@
"ts-node/register" "ts-node/register"
], ],
"args": [ "args": [
"${workspaceFolder}/index.ts" "${workspaceFolder}/src/index.ts"
] ]
} }
] ]

14
.vscode/settings.json vendored
View File

@@ -13,6 +13,16 @@
"[typescript]": { "[typescript]": {
"editor.formatOnSave": false, "editor.formatOnSave": false,
}, },
"eslint.enable": true, "[json]": {
"typescript.tsdk": "node_modules\\typescript\\lib", "files.insertFinalNewline": true
},
"typescript.tsdk": "node_modules/typescript/lib",
"sqltools.connections": [
{
"previewLimit": 50,
"driver": "SQLite",
"name": "Local SQLite",
"database": "./config/db/db.sqlite3"
}
],
} }

15
Dockerfile
View File

@@ -1,8 +1,8 @@
######## ########
# BASE # BASE
######## ########
FROM keymetrics/pm2:12-alpine as base FROM node:16-alpine3.14 as base
WORKDIR /usr/src/markbot WORKDIR /usr/src/app
COPY package*.json ./ COPY package*.json ./
# Install build tools for erlpack, then install prod deps only, then remove build tools # Install build tools for erlpack, then install prod deps only, then remove build tools
@@ -25,23 +25,20 @@ RUN npm run build
######## ########
# DEPLOY # DEPLOY
######## ########
FROM keymetrics/pm2:12-alpine as deploy FROM node:16-alpine3.14 as deploy
WORKDIR /usr/src/markbot WORKDIR /usr/src/app
ENV NPM_CONFIG_LOGLEVEL warn ENV NPM_CONFIG_LOGLEVEL warn
# Steal node_modules from base image # Steal node_modules from base image
COPY --from=base /usr/src/markbot/node_modules ./node_modules/ COPY --from=base /usr/src/app/node_modules ./node_modules/
# Steal compiled code from build image # Steal compiled code from build image
COPY --from=build /usr/src/markbot/dist ./ COPY --from=build /usr/src/app/dist ./
# Copy package.json for version number # Copy package.json for version number
COPY package*.json ./ COPY package*.json ./
# Copy PM2 config
COPY ecosystem.config.js .
RUN mkdir config RUN mkdir config
# RUN ls -al # RUN ls -al

4
config/.gitignore vendored
View File

@@ -1,4 +0,0 @@
# Ignore everything in this directory
*
# Except this file
!.gitignore

BIN
config/db/db.sqlite3 Normal file
View File

Binary file not shown.

BIN
config/db/db.sqlite3-shm Normal file
View File

Binary file not shown.

BIN
config/db/db.sqlite3-wal Normal file
View File

Binary file not shown.

12
ecosystem.config.js
View File

@@ -1,12 +0,0 @@
module.exports = {
apps: [{
name: 'markbot',
script: './index.js',
env: {
NODE_ENV: 'development',
},
env_production: {
NODE_ENV: 'production',
},
}],
};

420
index.ts
View File

@@ -1,420 +0,0 @@
/* eslint-disable no-console */
import 'source-map-support/register';
import * as Discord from 'discord.js';
// https://discord.js.org/#/docs/main/stable/general/welcome
import * as fs from 'fs';
import Markov, {
MarkovGenerateOptions,
MarkovResult,
MarkovConstructorOptions,
MarkovImportExport,
} from 'markov-strings';
import * as schedule from 'node-schedule';
interface MessageRecord {
id: string;
string: string;
attachment?: string;
}
interface MarkbotMarkovResult extends MarkovResult {
refs: Array<MessageRecord>;
}
interface MessagesDB {
messages: MessageRecord[];
}
interface MarkbotConfig {
stateSize?: number;
minScore?: number;
maxTries?: number;
prefix?: string;
game?: string;
token?: string;
}
const version: string = JSON.parse(fs.readFileSync('./package.json', 'utf8')).version || '0.0.0';
const client = new Discord.Client();
// const ZEROWIDTH_SPACE = String.fromCharCode(parseInt('200B', 16));
// const MAXMESSAGELENGTH = 2000;
const PAGE_SIZE = 100;
// let guilds = [];
// let connected = -1;
let GAME = '!mark help';
let PREFIX = '!mark';
let STATE_SIZE = 2; // Value of 1 to 3, based on corpus quality
let MAX_TRIES = 1000;
let MIN_SCORE = 10;
const inviteCmd = 'invite';
const errors: string[] = [];
let fileObj: MessagesDB = {
messages: [],
};
let markovDB: MessageRecord[] = [];
let messageCache: MessageRecord[] = [];
let deletionCache: string[] = [];
let markovOpts: MarkovConstructorOptions = {
stateSize: STATE_SIZE,
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
function uniqueBy<Record extends { [key: string]: any }>(
arr: Record[],
propertyName: keyof Record
): Record[] {
const unique: Record[] = [];
const found: { [key: string]: boolean } = {};
for (let i = 0; i < arr.length; i += 1) {
if (arr[i][propertyName]) {
const value = arr[i][propertyName];
if (!found[value]) {
found[value] = true;
unique.push(arr[i]);
}
}
}
return unique;
}
/**
* Regenerates the corpus and saves all cached changes to disk
*/
function regenMarkov(): void {
console.log('Regenerating Markov corpus...');
try {
fileObj = JSON.parse(fs.readFileSync('config/markovDB.json', 'utf8'));
} catch (err) {
console.log('No markovDB.json, starting with initial values');
fileObj = {
messages: [
{
id: '0',
string: '',
},
],
};
}
// console.log("MessageCache", messageCache)
markovDB = fileObj.messages;
markovDB = uniqueBy<MessageRecord>(markovDB.concat(messageCache), 'id');
deletionCache.forEach(id => {
const removeIndex = markovDB.map(item => item.id).indexOf(id);
// console.log('Remove Index:', removeIndex)
markovDB.splice(removeIndex, 1);
});
deletionCache = [];
const markov = new Markov(markovOpts);
fileObj.messages = markovDB;
// console.log("WRITING THE FOLLOWING DATA:")
// console.log(fileObj)
fs.writeFileSync('config/markovDB.json', JSON.stringify(fileObj), 'utf-8');
fileObj.messages = [];
messageCache = [];
markov.addData(markovDB);
fs.writeFileSync('config/markov.json', JSON.stringify(markov.export()));
console.log('Done regenerating Markov corpus.');
}
/**
* Loads the config settings from disk
*/
function loadConfig(): void {
// Move config if in legacy location
if (fs.existsSync('./config.json')) {
console.log('Copying config.json to new location in ./config');
fs.renameSync('./config.json', './config/config.json');
}
if (fs.existsSync('./markovDB.json')) {
console.log('Copying markovDB.json to new location in ./config');
fs.renameSync('./markovDB.json', './config/markovDB.json');
}
let token = 'missing';
try {
const cfg: MarkbotConfig = JSON.parse(fs.readFileSync('./config/config.json', 'utf8'));
PREFIX = cfg.prefix || '!mark';
GAME = cfg.game || '!mark help';
token = cfg.token || process.env.TOKEN || token;
STATE_SIZE = cfg.stateSize || STATE_SIZE;
MIN_SCORE = cfg.minScore || MIN_SCORE;
MAX_TRIES = cfg.maxTries || MAX_TRIES;
} catch (e) {
console.warn('Failed to read config.json.');
token = process.env.TOKEN || token;
}
try {
client.login(token);
} catch (e) {
console.error('Failed to login with token:', token);
}
markovOpts = {
stateSize: STATE_SIZE,
};
}
/**
* Checks if the author of a message as moderator-like permissions.
* @param {GuildMember} member Sender of the message
* @return {Boolean} True if the sender is a moderator.
*/
function isModerator(member: Discord.GuildMember): boolean {
return (
member.hasPermission('ADMINISTRATOR') ||
member.hasPermission('MANAGE_CHANNELS') ||
member.hasPermission('KICK_MEMBERS') ||
member.hasPermission('MOVE_MEMBERS') ||
member.id === '82684276755136512' // charlocharlie#8095
);
}
/**
* Reads a new message and checks if and which command it is.
* @param {Message} message Message to be interpreted as a command
* @return {String} Command string
*/
function validateMessage(message: Discord.Message): string | null {
const messageText = message.content.toLowerCase();
let command = null;
const thisPrefix = messageText.substring(0, PREFIX.length);
if (thisPrefix === PREFIX) {
const split = messageText.split(' ');
if (split[0] === PREFIX && split.length === 1) {
command = 'respond';
} else if (split[1] === 'train') {
command = 'train';
} else if (split[1] === 'help') {
command = 'help';
} else if (split[1] === 'regen') {
command = 'regen';
} else if (split[1] === 'invite') {
command = 'invite';
} else if (split[1] === 'debug') {
command = 'debug';
} else if (split[1] === 'tts') {
command = 'tts';
}
}
return command;
}
/**
* Function to recursively get all messages in a text channel's history. Ends
* by regnerating the corpus.
* @param {Message} message Message initiating the command, used for getting
* channel data
*/
async function fetchMessages(message: Discord.Message): Promise<void> {
let historyCache: MessageRecord[] = [];
let keepGoing = true;
let oldestMessageID: string | undefined;
while (keepGoing) {
const messages: Discord.Collection<
string,
Discord.Message
// eslint-disable-next-line no-await-in-loop
> = await message.channel.messages.fetch({
before: oldestMessageID,
limit: PAGE_SIZE,
});
const nonBotMessageFormatted = messages
.filter(elem => !elem.author.bot)
.map(elem => {
const dbObj: MessageRecord = {
string: elem.content,
id: elem.id,
};
if (elem.attachments.size > 0) {
dbObj.attachment = elem.attachments.values().next().value.url;
}
return dbObj;
});
historyCache = historyCache.concat(nonBotMessageFormatted);
const lastMessage = messages.last();
if (!lastMessage || messages.size < PAGE_SIZE) {
keepGoing = false;
} else {
oldestMessageID = lastMessage.id;
}
}
console.log(`Trained from ${historyCache.length} past human authored messages.`);
messageCache = messageCache.concat(historyCache);
regenMarkov();
message.reply(`Finished training from past ${historyCache.length} messages.`);
}
/**
* General Markov-chain response function
* @param {Message} message The message that invoked the action, used for channel info.
* @param {Boolean} debug Sends debug info as a message if true.
* @param {Boolean} tts If the message should be sent as TTS. Defaults to the TTS setting of the
* invoking message.
*/
function generateResponse(message: Discord.Message, debug = false, tts = message.tts): void {
console.log('Responding...');
const options: MarkovGenerateOptions = {
filter: (result): boolean => {
return result.score >= MIN_SCORE;
},
maxTries: MAX_TRIES,
};
const fsMarkov = new Markov();
const markovFile = JSON.parse(
fs.readFileSync('config/markov.json', 'utf-8')
) as MarkovImportExport;
fsMarkov.import(markovFile);
try {
const myResult = fsMarkov.generate(options) as MarkbotMarkovResult;
console.log('Generated Result:', myResult);
const messageOpts: Discord.MessageOptions = { tts };
const attachmentRefs = myResult.refs
.filter(ref => Object.prototype.hasOwnProperty.call(ref, 'attachment'))
.map(ref => ref.attachment as string);
if (attachmentRefs.length > 0) {
const randomRefAttachment = attachmentRefs[Math.floor(Math.random() * attachmentRefs.length)];
messageOpts.files = [randomRefAttachment];
} else {
const randomMessage = markovDB[Math.floor(Math.random() * markovDB.length)];
if (randomMessage.attachment) {
messageOpts.files = [{ attachment: randomMessage.attachment }];
}
}
myResult.string = myResult.string.replace(/@everyone/g, '@everyοne'); // Replace @everyone with a homoglyph 'o'
message.channel.send(myResult.string, messageOpts);
if (debug) message.channel.send(`\`\`\`\n${JSON.stringify(myResult, null, 2)}\n\`\`\``);
} catch (err) {
console.log(err);
if (debug) message.channel.send(`\n\`\`\`\nERROR: ${err}\n\`\`\``);
if (err.message.includes('Cannot build sentence with current corpus')) {
console.log('Not enough chat data for a response.');
}
}
}
client.on('ready', () => {
console.log('Markbot by Charlie Laabs');
if (client.user) client.user.setActivity(GAME);
regenMarkov();
});
client.on('error', err => {
const errText = `ERROR: ${err.name} - ${err.message}`;
console.log(errText);
errors.push(errText);
fs.writeFile('./config/error.json', JSON.stringify(errors), fsErr => {
if (fsErr) {
console.log(`error writing to error file: ${fsErr.message}`);
}
});
});
client.on('message', message => {
if (message.guild) {
const command = validateMessage(message);
if (command === 'help') {
const avatarURL = client.user?.avatarURL() || undefined;
const richem = new Discord.MessageEmbed()
.setAuthor(client.user?.username, avatarURL)
.setThumbnail(avatarURL as string)
.setDescription('A Markov chain chatbot that speaks based on previous chat input.')
.addField(
'!mark',
'Generates a sentence to say based on the chat database. Send your ' +
'message as TTS to recieve it as TTS.'
)
.addField(
'!mark train',
'Fetches the maximum amount of previous messages in the current ' +
'text channel, adds it to the database, and regenerates the corpus. Takes some time.'
)
.addField(
'!mark regen',
'Manually regenerates the corpus to add recent chat info. Run ' +
'this before shutting down to avoid any data loss. This automatically runs at midnight.'
)
.addField(
'!mark invite',
"Don't invite this bot to other servers. The database is shared " +
'between all servers and text channels.'
)
.addField('!mark debug', 'Runs the !mark command and follows it up with debug info.')
.setFooter(`Markov Discord v${version} by Charlie Laabs`);
message.channel.send(richem).catch(() => {
message.author.send(richem);
});
}
if (command === 'train') {
if (message.member && isModerator(message.member)) {
console.log('Training...');
fileObj = {
messages: [],
};
fs.writeFileSync('config/markovDB.json', JSON.stringify(fileObj), 'utf-8');
fetchMessages(message);
}
}
if (command === 'respond') {
generateResponse(message);
}
if (command === 'tts') {
generateResponse(message, false, true);
}
if (command === 'debug') {
generateResponse(message, true);
}
if (command === 'regen') {
regenMarkov();
}
if (command === null) {
console.log('Listening...');
if (!message.author.bot) {
const dbObj: MessageRecord = {
string: message.content,
id: message.id,
};
if (message.attachments.size > 0) {
dbObj.attachment = message.attachments.values().next().value.url;
}
messageCache.push(dbObj);
if (client.user && message.mentions.has(client.user)) {
generateResponse(message);
}
}
}
if (command === inviteCmd) {
const avatarURL = client.user?.avatarURL() || undefined;
const richem = new Discord.MessageEmbed()
.setAuthor(`Invite ${client.user?.username}`, avatarURL)
.setThumbnail(avatarURL as string)
.addField(
'Invite',
`[Invite ${client.user?.username} to your server](https://discordapp.com/oauth2/authorize?client_id=${client.user?.id}&scope=bot)`
);
message.channel.send(richem).catch(() => {
message.author.send(richem);
});
}
}
});
client.on('messageDelete', message => {
// console.log('Adding message ' + message.id + ' to deletion cache.')
deletionCache.push(message.id);
console.log('deletionCache:', deletionCache);
});
loadConfig();
schedule.scheduleJob('0 4 * * *', () => regenMarkov());

40
ormconfig.js Normal file
View File

@@ -0,0 +1,40 @@
const devConfig = {
type: 'better-sqlite3',
database: process.env.CONFIG_DIR
? `${process.env.CONFIG_DIR}/db/db.sqlite3`
: 'config/db/db.sqlite3',
synchronize: true,
migrationsRun: false,
// logging: true,
// logging: "all",
enableWAL: true,
entities: ['src/entity/**/*.ts'],
migrations: ['src/migration/**/*.ts'],
subscribers: ['src/subscriber/**/*.ts'],
cli: {
entitiesDir: 'src/entity',
migrationsDir: 'src/migration',
},
};
const prodConfig = {
type: 'better-sqlite3',
database: process.env.CONFIG_DIR
? `${process.env.CONFIG_DIR}/db/db.sqlite3`
: 'config/db/db.sqlite3',
synchronize: false,
logging: false,
enableWAL: true,
entities: ['dist/entity/**/*.js'],
migrations: ['dist/migration/**/*.js'],
migrationsRun: false,
subscribers: ['dist/subscriber/**/*.js'],
cli: {
entitiesDir: 'dist/entity',
migrationsDir: 'dist/migration',
},
};
const finalConfig = process.env.NODE_ENV !== 'production' ? devConfig : prodConfig;
module.exports = finalConfig;

7895
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

79
package.json
View File

@@ -1,16 +1,16 @@
{ {
"name": "markbot", "name": "markov-discord",
"version": "0.7.3", "version": "3.0.0",
"description": "A conversational Markov chain bot for Discord", "description": "A conversational Markov chain bot for Discord",
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"start": "node dist/index.js", "start": "node dist/index.js",
"start:ts": "ts-node index.ts", "start:ts": "ts-node src/index.ts",
"build": "tsc", "build": "rimraf dist && tsc",
"lint": "tsc --noEmit && eslint **/*.ts", "lint": "tsc --noEmit && eslint **/*.ts *.js",
"docker-up": "docker run --rm -e TOKEN=abc123 -it $(docker build -q .)" "docker-up": "docker run --rm -e TOKEN=abc123 -it $(docker build -q .)"
}, },
"repository": "https://github.com/charlocharlie/markov-discord.git", "repository": "https://github.com/claabs/markov-discord.git",
"keywords": [ "keywords": [
"discord", "discord",
"markov", "markov",
@@ -22,36 +22,51 @@
"markov-chain-bot", "markov-chain-bot",
"docker" "docker"
], ],
"author": "Charlie Laabs <charlielaabs@gmail.com>", "author": {
"name": "Charlie Laabs",
"url": "https://github.com/claabs"
},
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"bufferutil": "^4.0.1", "@discordjs/builders": "^0.9.0",
"discord.js": "^12.3.1", "@discordjs/rest": "^0.2.0-canary.0",
"erlpack": "^0.1.3", "@types/fs-extra": "^9.0.13",
"markov-strings": "^3.0.1", "better-sqlite3": "^7.4.5",
"node-schedule": "^1.3.2", "bufferutil": "^4.0.5",
"source-map-support": "^0.5.16", "class-transformer": "^0.5.1",
"class-validator": "^0.13.2",
"discord-api-types": "^0.25.2",
"discord.js": "^13.3.1",
"dotenv": "^10.0.0",
"erlpack": "github:discord/erlpack",
"fs-extra": "^10.0.0",
"json5": "^2.2.0",
"markov-strings-db": "file:../markov-strings/markov-strings-db-4.0.0.tgz",
"pino": "^7.5.1",
"pino-pretty": "^7.3.0",
"reflect-metadata": "^0.1.13",
"source-map-support": "^0.5.21",
"typeorm": "^0.2.38",
"utf-8-validate": "^5.0.7",
"zlib-sync": "^0.1.7" "zlib-sync": "^0.1.7"
}, },
"engines": {
"node": ">=12.0.0"
},
"devDependencies": { "devDependencies": {
"@types/node": "^12.12.21", "@types/node": "^16.11.14",
"@types/node-schedule": "^1.3.0", "@types/validator": "^13.7.0",
"@types/ws": "^7.2.7", "@typescript-eslint/eslint-plugin": "^5.7.0",
"@typescript-eslint/eslint-plugin": "^2.13.0", "@typescript-eslint/parser": "^5.7.0",
"@typescript-eslint/parser": "^2.13.0", "eslint": "^8.4.1",
"eslint": "^6.8.0", "eslint-config-airbnb-base": "^15.0.0",
"eslint-config-airbnb-base": "^14.0.0", "eslint-config-prettier": "^8.3.0",
"eslint-config-prettier": "^6.7.0", "eslint-plugin-import": "^2.25.3",
"eslint-plugin-import": "^2.19.1", "eslint-plugin-prettier": "^4.0.0",
"eslint-plugin-prettier": "^3.1.2", "prettier": "^2.5.1",
"prettier": "^1.19.1", "rimraf": "^3.0.2",
"ts-node": "^8.5.4", "ts-node": "^10.4.0",
"typescript": "^3.7.4" "types-package-json": "^2.0.39",
"typescript": "^4.5.4"
}, },
"eslintIgnore": [ "engines": {
"**/*.js" "node": "16"
] }
} }

131
src/config/classes.ts Normal file
View File

@@ -0,0 +1,131 @@
/* eslint-disable @typescript-eslint/no-empty-function, no-useless-constructor, max-classes-per-file */
import 'reflect-metadata';
import { Type } from 'class-transformer';
import { IsString, IsOptional, IsEnum, IsArray, IsInt, IsDefined } from 'class-validator';
export enum LogLevel {
SILENT = 'silent',
ERROR = 'error',
WARN = 'warn',
INFO = 'info',
DEBUG = 'debug',
TRACE = 'trace',
}
/**
* @example ```jsonc
* {
* "token": "k5NzE2NDg1MTIwMjc0ODQ0Nj.DSnXwg.ttNotARealToken5p3WfDoUxhiH",
* "commandPrefix": "!mark",
* "activity": "\"!mark help\" for help",
* "ownerIds": ["00000000000000000"],
* "logLevel": "info",
* }
* ```
*/
export class AppConfig {
/**
* Your Discord bot token
* @example k5NzE2NDg1MTIwMjc0ODQ0Nj.DSnXwg.ttNotARealToken5p3WfDoUxhiH
* @env TOKEN
*/
@IsDefined()
@IsString()
token = process.env.TOKEN as string;
/**
* The command prefix used to trigger the bot commands (when not using slash commands)
* @example !bot
* @default !mark
* @env CRON_SCHEDULE
*/
@IsOptional()
@IsString()
commandPrefix = process.env.COMMAND_PREFIX || '!mark';
/**
* The activity status shown under the bot's name in the user list
* @example "!mark help" for help
* @default !mark help
* @env ACTIVITY
*/
@IsOptional()
@IsString()
activity = process.env.ACTIVITY || '!mark help';
/**
* A list of Discord user IDs that have owner permissions for the bot
* @example ['82684276755136512']
* @default []
* @env OWNER_IDS (comma separated)
*/
@IsArray()
@IsString({ each: true })
@Type(() => String)
@IsOptional()
ownerIds = process.env.OWNER_IDS ? process.env.OWNER_IDS.split(',').map((id) => id.trim()) : [];
/**
* TZ name from this list: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List
* @example America/Chicago
* @default UTC
* @env TZ
*/
@IsOptional()
@IsString()
timezone = process.env.TZ || 'UTC';
/**
* Log level in lower case. Can be [silent, error, warn, info, debug, trace]
* @example debug
* @default info
* @env LOG_LEVEL
*/
@IsOptional()
@IsEnum(LogLevel)
logLevel = process.env.LOG_LEVEL || LogLevel.INFO;
/**
* The stateSize is the number of words for each "link" of the generated sentence.
* 1 will output gibberish sentences without much sense.
* 2 is a sensible default for most cases.
* 3 and more can create good sentences if you have a corpus that allows it.
* @example 3
* @default 2
* @env STATE_SIZE
*/
@IsOptional()
@IsInt()
stateSize = process.env.STATE_SIZE ? parseInt(process.env.STATE_SIZE, 10) : 2;
/**
* The number of tries the sentence generator will try before giving up
* @example 2000
* @default 1000
* @env MAX_TRIES
*/
@IsOptional()
@IsInt()
maxTries = process.env.MAX_TRIES ? parseInt(process.env.MAX_TRIES, 10) : 1000;
/**
* The minimum score required when generating a sentence.
* A relative "score" based on the number of possible permutations.
* Higher is "better", but the actual value depends on your corpus.
* @example 15
* @default 10
* @env MIN_SCORE
*/
@IsOptional()
@IsInt()
minScore = process.env.MIN_SCORE ? parseInt(process.env.MIN_SCORE, 10) : 10;
/**
* This guild ID should be declared if you want its commands to update immediately during development
* @example 1234567890
* @env DEV_GUILD_ID
*/
@IsOptional()
@IsString()
devGuildId = process.env.DEV_GUILD_ID;
}

2
src/config/index.ts Normal file
View File

@@ -0,0 +1,2 @@
export * from './classes';
export * from './setup';

80
src/config/setup.ts Normal file
View File

@@ -0,0 +1,80 @@
import 'reflect-metadata';
import { config as dotenv } from 'dotenv';
import json5 from 'json5';
import path from 'path';
import fs from 'fs-extra';
import { validateSync } from 'class-validator';
import { instanceToPlain, plainToInstance } from 'class-transformer';
import pino from 'pino';
import { AppConfig } from './classes';
dotenv();
// Declare pino logger as importing would cause dependency cycle
const L = pino({
transport: {
target: 'pino-pretty',
options: {
translateTime: `SYS:standard`,
},
},
formatters: {
level: (label) => {
return { level: label };
},
},
level: process.env.LOG_LEVEL || 'info',
base: undefined,
});
// TODO: Add YAML parser
const EXTENSIONS = ['.json', '.json5']; // Allow .json or .json5 extension
const removeFileExtension = (filename: string): string => {
const ext = path.extname(filename);
if (EXTENSIONS.includes(ext)) {
return path.basename(filename, ext);
}
return path.basename(filename);
};
export const CONFIG_DIR = process.env.CONFIG_DIR || 'config';
export const CONFIG_FILE_NAME = process.env.CONFIG_FILE_NAME
? removeFileExtension(process.env.CONFIG_FILE_NAME)
: 'config';
const configPaths = EXTENSIONS.map((ext) => path.resolve(CONFIG_DIR, `${CONFIG_FILE_NAME}${ext}`));
const configPath = configPaths.find((p) => fs.existsSync(p));
// eslint-disable-next-line import/no-mutable-exports
let config: AppConfig;
if (!configPath) {
L.warn('No config file detected');
const newConfigPath = path.resolve(CONFIG_DIR, `${CONFIG_FILE_NAME}.json`);
config = new AppConfig();
try {
L.debug({ newConfigPath }, 'Creating new config file');
fs.writeJSONSync(newConfigPath, instanceToPlain(config), { spaces: 2 });
L.info({ newConfigPath }, 'Wrote new default config file');
} catch (err) {
L.debug(err);
L.info('Not allowed to create new config. Continuing...');
}
} else {
L.debug({ configPath });
const parsedConfig = json5.parse(fs.readFileSync(configPath, 'utf8'));
config = plainToInstance(AppConfig, parsedConfig);
}
const errors = validateSync(config, {
validationError: {
target: false,
},
});
if (errors.length > 0) {
L.error({ errors }, 'Validation error(s)');
throw new Error('Invalid config');
}
L.debug({ config: instanceToPlain(config) });
export { config };

22
src/deploy-commands.ts Normal file
View File

@@ -0,0 +1,22 @@
import { SlashCommandBuilder } from '@discordjs/builders';
import { REST } from '@discordjs/rest';
import { Routes } from 'discord-api-types/v9';
import { config } from './config';
import { packageJson } from './util';
const helpSlashCommand = new SlashCommandBuilder()
.setName('help')
.setDescription(`How to use ${packageJson().name}`);
const commands = [helpSlashCommand.toJSON()];
export async function deployCommands(clientId: string) {
const rest = new REST({ version: '9' }).setToken(config.token);
if (config.devGuildId) {
await rest.put(Routes.applicationGuildCommands(clientId, config.devGuildId), {
body: commands,
});
} else {
await rest.put(Routes.applicationCommands(clientId), { body: commands });
}
}

17
src/entity/Channel.ts Normal file
View File

@@ -0,0 +1,17 @@
/* eslint-disable import/no-cycle */
import { PrimaryColumn, Entity, ManyToOne, BaseEntity, Column } from 'typeorm';
import { Guild } from './Guild';
@Entity()
export class Channel extends BaseEntity {
@PrimaryColumn()
id: string;
@Column({
default: true,
})
listen: boolean;
@ManyToOne(() => Guild, (guild) => guild.channels)
guild: Guild;
}

12
src/entity/Guild.ts Normal file
View File

@@ -0,0 +1,12 @@
/* eslint-disable import/no-cycle */
import { BaseEntity, Entity, OneToMany, PrimaryColumn } from 'typeorm';
import { Channel } from './Channel';
@Entity()
export class Guild extends BaseEntity {
@PrimaryColumn()
id: string;
@OneToMany(() => Channel, (channel) => channel.guild, { onDelete: 'CASCADE', cascade: true })
channels: Channel[];
}

390
src/index.ts Normal file
View File

@@ -0,0 +1,390 @@
/* eslint-disable no-console */
import 'source-map-support/register';
import 'reflect-metadata';
import * as Discord from 'discord.js';
import Markov, {
MarkovGenerateOptions,
MarkovConstructorOptions,
AddDataProps,
} from 'markov-strings-db';
import { createConnection } from 'typeorm';
import { MarkovInputData } from 'markov-strings-db/dist/src/entity/MarkovInputData';
import { APIInteractionGuildMember } from 'discord-api-types';
import L from './logger';
import { Channel } from './entity/Channel';
import { Guild } from './entity/Guild';
import { config } from './config';
import { deployCommands } from './deploy-commands';
import { getRandomElement, getVersion, packageJson } from './util';
interface MarkovDataCustom {
attachments: string[];
}
const client = new Discord.Client<true>({
intents: [Discord.Intents.FLAGS.GUILD_MESSAGES, Discord.Intents.FLAGS.GUILDS],
presence: {
activities: [
{
type: 'PLAYING',
name: config.activity,
url: packageJson().homepage,
},
],
},
});
const markovOpts: MarkovConstructorOptions = {
stateSize: config.stateSize,
};
/**
* #v3-complete
*/
async function getMarkovByGuildId(guildId: string): Promise<Markov> {
const id = parseInt(guildId, 10);
const markov = new Markov({ id, options: markovOpts });
await markov.setup(); // Connect the markov instance to the DB to assign it an ID
return markov;
}
/**
* Checks if the author of a message as moderator-like permissions.
* @param {GuildMember} member Sender of the message
* @return {Boolean} True if the sender is a moderator.
* #v3-complete
*/
function isModerator(member: Discord.GuildMember | APIInteractionGuildMember | null): boolean {
const MODERATOR_PERMISSIONS: Discord.PermissionResolvable[] = [
'ADMINISTRATOR',
'MANAGE_CHANNELS',
'KICK_MEMBERS',
'MOVE_MEMBERS',
];
if (!member) return false;
if (member instanceof Discord.GuildMember) {
return (
MODERATOR_PERMISSIONS.some((p) => member.permissions.has(p)) ||
config.ownerIds.includes(member.id)
);
}
// TODO: How to parse API permissions?
L.debug({ permissions: member.permissions });
return true;
}
type MessageCommands = 'respond' | 'train' | 'help' | 'invite' | 'debug' | 'tts' | null;
/**
* Reads a new message and checks if and which command it is.
* @param {Message} message Message to be interpreted as a command
* @return {String} Command string
*/
function validateMessage(message: Discord.Message): MessageCommands {
const messageText = message.content.toLowerCase();
let command: MessageCommands = null;
const thisPrefix = messageText.substring(0, config.commandPrefix.length);
if (thisPrefix === config.commandPrefix) {
const split = messageText.split(' ');
if (split[0] === config.commandPrefix && split.length === 1) {
command = 'respond';
} else if (split[1] === 'train') {
command = 'train';
} else if (split[1] === 'help') {
command = 'help';
} else if (split[1] === 'invite') {
command = 'invite';
} else if (split[1] === 'debug') {
command = 'debug';
} else if (split[1] === 'tts') {
command = 'tts';
}
}
return command;
}
function messageToData(message: Discord.Message): AddDataProps {
const attachmentUrls = message.attachments.map((a) => a.url);
let custom: MarkovDataCustom | undefined;
if (attachmentUrls.length) custom = { attachments: attachmentUrls };
return {
string: message.content,
custom,
};
}
/**
* Recursively gets all messages in a text channel's history.
* #v3-complete
*/
async function saveChannelMessageHistory(
channel: Discord.TextChannel,
interaction: Discord.Message | Discord.CommandInteraction
): Promise<void> {
if (!isModerator(interaction.member as any)) return;
const markov = await getMarkovByGuildId(channel.guildId);
L.debug({ channelId: channel.id }, `Training from text channel`);
const PAGE_SIZE = 100;
let keepGoing = true;
let oldestMessageID: string | undefined;
let channelMessagesCount = 0;
while (keepGoing) {
// eslint-disable-next-line no-await-in-loop
const messages = await channel.messages.fetch({
before: oldestMessageID,
limit: PAGE_SIZE,
});
const nonBotMessageFormatted = messages.filter((elem) => !elem.author.bot).map(messageToData);
L.debug({ oldestMessageID }, `Saving ${nonBotMessageFormatted.length} messages`);
// eslint-disable-next-line no-await-in-loop
await markov.addData(nonBotMessageFormatted);
L.trace('Finished saving messages');
channelMessagesCount += nonBotMessageFormatted.length;
const lastMessage = messages.last();
if (!lastMessage || messages.size < PAGE_SIZE) {
keepGoing = false;
} else {
oldestMessageID = lastMessage.id;
}
}
L.info(
{ channelId: channel.id },
`Trained from ${channelMessagesCount} past human authored messages.`
);
await interaction.reply(`Trained from ${channelMessagesCount} past human authored messages.`);
}
/**
* General Markov-chain response function
* @param interaction The message that invoked the action, used for channel info.
* @param debug Sends debug info as a message if true.
* @param tts If the message should be sent as TTS. Defaults to the TTS setting of the
* invoking message.
* #v3-complete
*/
async function generateResponse(
interaction: Discord.Message | Discord.CommandInteraction,
debug = false,
tts = false
): Promise<void> {
L.debug('Responding...');
if (!interaction.guildId) {
L.info('Received a message without a guildId');
return;
}
const options: MarkovGenerateOptions<MarkovDataCustom> = {
filter: (result): boolean => {
return result.score >= config.minScore;
},
maxTries: config.maxTries,
};
const markov = await getMarkovByGuildId(interaction.guildId);
try {
const response = await markov.generate<MarkovDataCustom>(options);
L.info({ response }, 'Generated response');
const messageOpts: Discord.MessageOptions = { tts };
const attachmentUrls = response.refs
.filter((ref) => ref.custom && 'attachments' in ref.custom)
.flatMap((ref) => ref.custom.attachments);
if (attachmentUrls.length > 0) {
const randomRefAttachment = getRandomElement(attachmentUrls);
messageOpts.files = [randomRefAttachment];
} else {
// TODO: This might not even work
const randomMessage = await MarkovInputData.createQueryBuilder<
MarkovInputData<MarkovDataCustom>
>('input')
.leftJoinAndSelect('input.fragment', 'fragment')
.leftJoinAndSelect('fragment.corpusEntry', 'corpusEntry')
.where([
{
fragment: { startWordMarkov: markov.db },
},
{
fragment: { endWordMarkov: markov.db },
},
{
fragment: { corpusEntry: { markov: markov.db } },
},
])
.orderBy('RANDOM()')
.limit(1)
.getOne();
const randomMessageAttachmentUrls = randomMessage?.custom?.attachments;
if (randomMessageAttachmentUrls?.length) {
messageOpts.files = [{ attachment: getRandomElement(randomMessageAttachmentUrls) }];
}
}
response.string = response.string.replace(/@everyone/g, '@everyοne'); // Replace @everyone with a homoglyph 'o'
messageOpts.content = response.string;
if (interaction instanceof Discord.Message) {
await interaction.channel.send(messageOpts);
if (debug) {
await interaction.channel.send(`\`\`\`\n${JSON.stringify(response, null, 2)}\n\`\`\``);
}
} else if (interaction instanceof Discord.CommandInteraction) {
await interaction.editReply(messageOpts);
if (debug) {
await interaction.followUp(`\`\`\`\n${JSON.stringify(response, null, 2)}\n\`\`\``);
}
}
} catch (err) {
L.error(err);
if (debug) {
if (interaction instanceof Discord.Message) {
await interaction.channel.send(`\n\`\`\`\nERROR: ${err}\n\`\`\``);
} else if (interaction instanceof Discord.CommandInteraction) {
await interaction.editReply(`\n\`\`\`\nERROR: ${err}\n\`\`\``);
}
}
}
}
function helpMessage(): Discord.MessageOptions {
const avatarURL = client.user.avatarURL() || undefined;
const embed = new Discord.MessageEmbed()
.setAuthor(client.user.username || packageJson().name, avatarURL)
.setThumbnail(avatarURL as string)
.setDescription('A Markov chain chatbot that speaks based on previous chat input.')
.addField(
`${config.commandPrefix}`,
'Generates a sentence to say based on the chat database. Send your ' +
'message as TTS to recieve it as TTS.'
)
.addField(
`${config.commandPrefix} train`,
'Fetches the maximum amount of previous messages in the current ' +
'text channel, adds it to the database, and regenerates the corpus. Takes some time.'
)
.addField(
`${config.commandPrefix} invite`,
"Don't invite this bot to other servers. The database is shared " +
'between all servers and text channels.'
)
.addField(
`${config.commandPrefix} debug`,
`Runs the ${config.commandPrefix} command and follows it up with debug info.`
)
.setFooter(`Markov Discord ${getVersion()} by ${packageJson().author}`);
return {
embeds: [embed],
};
}
function inviteMessage(): Discord.MessageOptions {
const avatarURL = client.user.avatarURL() || undefined;
const embed = new Discord.MessageEmbed()
.setAuthor(`Invite ${client.user?.username}`, avatarURL)
.setThumbnail(avatarURL as string)
.addField(
'Invite',
`[Invite ${client.user.username} to your server](https://discord.com/api/oauth2/authorize?client_id=${client.user.id}&permissions=105472&scope=bot%20applications.commands)`
);
return { embeds: [embed] };
}
client.on('ready', async (readyClient) => {
L.info('Bot logged in');
await deployCommands(readyClient.user.id);
const guildsToSave: Guild[] = [];
const channelsToSave: Channel[] = [];
readyClient.guilds.valueOf().forEach((guild) => {
const dbGuild = Guild.create({ id: guild.id });
const textChannels = guild.channels.valueOf().filter((channel) => channel.isText());
const dbChannels = textChannels.map((channel) =>
Channel.create({ id: channel.id, guild: dbGuild })
);
guildsToSave.push(dbGuild);
channelsToSave.push(...dbChannels);
});
await Guild.upsert(guildsToSave, ['id']);
await Channel.upsert(channelsToSave, ['id']); // TODO: ensure this doesn't overwrite the existing `listen`
});
client.on('error', (err) => {
L.error(err);
});
client.on('messageCreate', async (message) => {
if (!(message.guild && message.channel instanceof Discord.TextChannel)) return;
const command = validateMessage(message);
if (command === 'help') {
await message.channel.send(helpMessage());
}
if (command === 'invite') {
await message.channel.send(inviteMessage());
}
if (command === 'train') {
await saveChannelMessageHistory(message.channel, message);
}
if (command === 'respond') {
await generateResponse(message);
}
if (command === 'tts') {
await generateResponse(message, false, true);
}
if (command === 'debug') {
await generateResponse(message, true);
}
if (command === null) {
L.debug('Listening...');
if (!message.author.bot) {
const markov = await getMarkovByGuildId(message.channel.guildId);
await markov.addData([messageToData(message)]);
if (client.user && message.mentions.has(client.user)) {
await generateResponse(message);
}
}
}
});
/**
* #v3-complete
*/
client.on('messageDelete', async (message) => {
L.info(`Deleting message ${message.id}`);
if (!(message.guildId && message.content)) {
return;
}
const markov = await getMarkovByGuildId(message.guildId);
await markov.removeData([message.content]);
});
/**
* #v3-complete
*/
client.on('messageUpdate', async (oldMessage, newMessage) => {
L.info(`Editing message ${oldMessage.id}`);
if (!(oldMessage.guildId && oldMessage.content && newMessage.content)) {
return;
}
const markov = await getMarkovByGuildId(oldMessage.guildId);
await markov.removeData([oldMessage.content]);
await markov.addData([newMessage.content]);
});
/**
* Loads the config settings from disk
*/
async function main(): Promise<void> {
const connection = await Markov.extendConnectionOptions();
await createConnection(connection);
await client.login(config.token);
// Move config if in legacy location
// TODO: import legacy DB?
}
main();

22
src/logger.ts Normal file
View File

@@ -0,0 +1,22 @@
import pino from 'pino';
import dotenv from 'dotenv';
dotenv.config();
const logger = pino({
transport: {
target: 'pino-pretty',
options: {
translateTime: `SYS:standard`,
},
},
formatters: {
level: (label) => {
return { level: label };
},
},
level: process.env.LOG_LEVEL || 'info',
base: undefined,
});
export default logger;

0
src/migration/.gitkeep Normal file
View File

0
src/subscriber/.gitkeep Normal file
View File

21
src/util.ts Normal file
View File

@@ -0,0 +1,21 @@
import fs from 'fs-extra';
import path from 'path';
import type { PackageJson } from 'types-package-json';
let packageJsonCache: PackageJson | undefined;
export const packageJson = (): PackageJson => {
if (packageJsonCache) return packageJsonCache;
packageJsonCache = fs.readJSONSync(path.resolve(process.cwd(), `package.json`));
return packageJsonCache as PackageJson;
};
export const getVersion = (): string => {
const { COMMIT_SHA } = process.env;
let { version } = packageJson();
if (COMMIT_SHA) version = `${version}#${COMMIT_SHA.substring(0, 8)}`;
return version;
};
export const getRandomElement = <T>(array: T[]): T => {
return array[Math.floor(Math.random() * array.length)];
};

10
tsconfig.json
View File

@@ -4,17 +4,19 @@
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
"outDir": "./dist", /* Redirect output structure to the directory. */ "outDir": "./dist", /* Redirect output structure to the directory. */
"removeComments": true, /* Do not emit comments to output. */ "removeComments": true, /* Do not emit comments to output. */
"esModuleInterop": true,
"strict": true, /* Enable all strict type-checking options. */ "strict": true, /* Enable all strict type-checking options. */
"moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
"typeRoots": [ "typeRoots": [
"node_modules/@types" "node_modules/@types"
], /* List of folders to include type definitions from. */ ], /* List of folders to include type definitions from. */
"inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ "forceConsistentCasingInFileNames": true, /* Disallow inconsistently-cased references to the same file. */
"useUnknownInCatchVariables": false,
"experimentalDecorators": true,
"strictPropertyInitialization": false,
"emitDecoratorMetadata": true,
}, },
"include": [
"**/*.ts"
],
"exclude": [ "exclude": [
"node_modules", "node_modules",
"dist" "dist"