Initial rough working draft of v3

src/config/classes.ts

@@ -0,0 +1,131 @@
+/* eslint-disable @typescript-eslint/no-empty-function, no-useless-constructor, max-classes-per-file */
+import 'reflect-metadata';
+import { Type } from 'class-transformer';
+import { IsString, IsOptional, IsEnum, IsArray, IsInt, IsDefined } from 'class-validator';
+
+export enum LogLevel {
+  SILENT = 'silent',
+  ERROR = 'error',
+  WARN = 'warn',
+  INFO = 'info',
+  DEBUG = 'debug',
+  TRACE = 'trace',
+}
+
+/**
+ * @example ```jsonc
+ * {
+ *   "token": "k5NzE2NDg1MTIwMjc0ODQ0Nj.DSnXwg.ttNotARealToken5p3WfDoUxhiH",
+ *   "commandPrefix": "!mark",
+ *   "activity": "\"!mark help\" for help",
+ *   "ownerIds": ["00000000000000000"],
+ *   "logLevel": "info",
+ * }
+ * ```
+ */
+export class AppConfig {
+  /**
+   * Your Discord bot token
+   * @example k5NzE2NDg1MTIwMjc0ODQ0Nj.DSnXwg.ttNotARealToken5p3WfDoUxhiH
+   * @env TOKEN
+   */
+  @IsDefined()
+  @IsString()
+  token = process.env.TOKEN as string;
+
+  /**
+   * The command prefix used to trigger the bot commands (when not using slash commands)
+   * @example !bot
+   * @default !mark
+   * @env CRON_SCHEDULE
+   */
+  @IsOptional()
+  @IsString()
+  commandPrefix = process.env.COMMAND_PREFIX || '!mark';
+
+  /**
+   * The activity status shown under the bot's name in the user list
+   * @example "!mark help" for help
+   * @default !mark help
+   * @env ACTIVITY
+   */
+  @IsOptional()
+  @IsString()
+  activity = process.env.ACTIVITY || '!mark help';
+
+  /**
+   * A list of Discord user IDs that have owner permissions for the bot
+   * @example ['82684276755136512']
+   * @default []
+   * @env OWNER_IDS (comma separated)
+   */
+  @IsArray()
+  @IsString({ each: true })
+  @Type(() => String)
+  @IsOptional()
+  ownerIds = process.env.OWNER_IDS ? process.env.OWNER_IDS.split(',').map((id) => id.trim()) : [];
+
+  /**
+   * TZ name from this list: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List
+   * @example America/Chicago
+   * @default UTC
+   * @env TZ
+   */
+  @IsOptional()
+  @IsString()
+  timezone = process.env.TZ || 'UTC';
+
+  /**
+   * Log level in lower case. Can be [silent, error, warn, info, debug, trace]
+   * @example debug
+   * @default info
+   * @env LOG_LEVEL
+   */
+  @IsOptional()
+  @IsEnum(LogLevel)
+  logLevel = process.env.LOG_LEVEL || LogLevel.INFO;
+
+  /**
+   * The stateSize is the number of words for each "link" of the generated sentence.
+   * 1 will output gibberish sentences without much sense.
+   * 2 is a sensible default for most cases.
+   * 3 and more can create good sentences if you have a corpus that allows it.
+   * @example 3
+   * @default 2
+   * @env STATE_SIZE
+   */
+  @IsOptional()
+  @IsInt()
+  stateSize = process.env.STATE_SIZE ? parseInt(process.env.STATE_SIZE, 10) : 2;
+
+  /**
+   * The number of tries the sentence generator will try before giving up
+   * @example 2000
+   * @default 1000
+   * @env MAX_TRIES
+   */
+  @IsOptional()
+  @IsInt()
+  maxTries = process.env.MAX_TRIES ? parseInt(process.env.MAX_TRIES, 10) : 1000;
+
+  /**
+   * The minimum score required when generating a sentence.
+   * A relative "score" based on the number of possible permutations.
+   * Higher is "better", but the actual value depends on your corpus.
+   * @example 15
+   * @default 10
+   * @env MIN_SCORE
+   */
+  @IsOptional()
+  @IsInt()
+  minScore = process.env.MIN_SCORE ? parseInt(process.env.MIN_SCORE, 10) : 10;
+
+  /**
+   * This guild ID should be declared if you want its commands to update immediately during development
+   * @example 1234567890
+   * @env DEV_GUILD_ID
+   */
+  @IsOptional()
+  @IsString()
+  devGuildId = process.env.DEV_GUILD_ID;
+}

src/config/index.ts

@@ -0,0 +1,2 @@
+export * from './classes';
+export * from './setup';

src/config/setup.ts

@@ -0,0 +1,80 @@
+import 'reflect-metadata';
+import { config as dotenv } from 'dotenv';
+import json5 from 'json5';
+import path from 'path';
+import fs from 'fs-extra';
+import { validateSync } from 'class-validator';
+import { instanceToPlain, plainToInstance } from 'class-transformer';
+import pino from 'pino';
+import { AppConfig } from './classes';
+
+dotenv();
+
+// Declare pino logger as importing would cause dependency cycle
+const L = pino({
+  transport: {
+    target: 'pino-pretty',
+    options: {
+      translateTime: `SYS:standard`,
+    },
+  },
+  formatters: {
+    level: (label) => {
+      return { level: label };
+    },
+  },
+  level: process.env.LOG_LEVEL || 'info',
+  base: undefined,
+});
+
+// TODO: Add YAML parser
+const EXTENSIONS = ['.json', '.json5']; // Allow .json or .json5 extension
+
+const removeFileExtension = (filename: string): string => {
+  const ext = path.extname(filename);
+  if (EXTENSIONS.includes(ext)) {
+    return path.basename(filename, ext);
+  }
+  return path.basename(filename);
+};
+
+export const CONFIG_DIR = process.env.CONFIG_DIR || 'config';
+export const CONFIG_FILE_NAME = process.env.CONFIG_FILE_NAME
+  ? removeFileExtension(process.env.CONFIG_FILE_NAME)
+  : 'config';
+
+const configPaths = EXTENSIONS.map((ext) => path.resolve(CONFIG_DIR, `${CONFIG_FILE_NAME}${ext}`));
+const configPath = configPaths.find((p) => fs.existsSync(p));
+// eslint-disable-next-line import/no-mutable-exports
+let config: AppConfig;
+if (!configPath) {
+  L.warn('No config file detected');
+  const newConfigPath = path.resolve(CONFIG_DIR, `${CONFIG_FILE_NAME}.json`);
+  config = new AppConfig();
+  try {
+    L.debug({ newConfigPath }, 'Creating new config file');
+    fs.writeJSONSync(newConfigPath, instanceToPlain(config), { spaces: 2 });
+    L.info({ newConfigPath }, 'Wrote new default config file');
+  } catch (err) {
+    L.debug(err);
+    L.info('Not allowed to create new config. Continuing...');
+  }
+} else {
+  L.debug({ configPath });
+  const parsedConfig = json5.parse(fs.readFileSync(configPath, 'utf8'));
+  config = plainToInstance(AppConfig, parsedConfig);
+}
+
+const errors = validateSync(config, {
+  validationError: {
+    target: false,
+  },
+});
+if (errors.length > 0) {
+  L.error({ errors }, 'Validation error(s)');
+  throw new Error('Invalid config');
+}
+
+L.debug({ config: instanceToPlain(config) });
+
+export { config };

src/deploy-commands.ts

@@ -0,0 +1,22 @@
+import { SlashCommandBuilder } from '@discordjs/builders';
+import { REST } from '@discordjs/rest';
+import { Routes } from 'discord-api-types/v9';
+import { config } from './config';
+import { packageJson } from './util';
+
+const helpSlashCommand = new SlashCommandBuilder()
+  .setName('help')
+  .setDescription(`How to use ${packageJson().name}`);
+
+const commands = [helpSlashCommand.toJSON()];
+
+export async function deployCommands(clientId: string) {
+  const rest = new REST({ version: '9' }).setToken(config.token);
+  if (config.devGuildId) {
+    await rest.put(Routes.applicationGuildCommands(clientId, config.devGuildId), {
+      body: commands,
+    });
+  } else {
+    await rest.put(Routes.applicationCommands(clientId), { body: commands });
+  }
+}

src/entity/Channel.ts

@@ -0,0 +1,17 @@
+/* eslint-disable import/no-cycle */
+import { PrimaryColumn, Entity, ManyToOne, BaseEntity, Column } from 'typeorm';
+import { Guild } from './Guild';
+
+@Entity()
+export class Channel extends BaseEntity {
+  @PrimaryColumn()
+  id: string;
+
+  @Column({
+    default: true,
+  })
+  listen: boolean;
+
+  @ManyToOne(() => Guild, (guild) => guild.channels)
+  guild: Guild;
+}

src/entity/Guild.ts

@@ -0,0 +1,12 @@
+/* eslint-disable import/no-cycle */
+import { BaseEntity, Entity, OneToMany, PrimaryColumn } from 'typeorm';
+import { Channel } from './Channel';
+
+@Entity()
+export class Guild extends BaseEntity {
+  @PrimaryColumn()
+  id: string;
+
+  @OneToMany(() => Channel, (channel) => channel.guild, { onDelete: 'CASCADE', cascade: true })
+  channels: Channel[];
+}

src/index.ts

@@ -0,0 +1,390 @@
+/* eslint-disable no-console */
+import 'source-map-support/register';
+import 'reflect-metadata';
+import * as Discord from 'discord.js';
+
+import Markov, {
+  MarkovGenerateOptions,
+  MarkovConstructorOptions,
+  AddDataProps,
+} from 'markov-strings-db';
+
+import { createConnection } from 'typeorm';
+import { MarkovInputData } from 'markov-strings-db/dist/src/entity/MarkovInputData';
+import { APIInteractionGuildMember } from 'discord-api-types';
+import L from './logger';
+import { Channel } from './entity/Channel';
+import { Guild } from './entity/Guild';
+import { config } from './config';
+import { deployCommands } from './deploy-commands';
+import { getRandomElement, getVersion, packageJson } from './util';
+
+interface MarkovDataCustom {
+  attachments: string[];
+}
+
+const client = new Discord.Client<true>({
+  intents: [Discord.Intents.FLAGS.GUILD_MESSAGES, Discord.Intents.FLAGS.GUILDS],
+  presence: {
+    activities: [
+      {
+        type: 'PLAYING',
+        name: config.activity,
+        url: packageJson().homepage,
+      },
+    ],
+  },
+});
+
+const markovOpts: MarkovConstructorOptions = {
+  stateSize: config.stateSize,
+};
+
+/**
+ * #v3-complete
+ */
+async function getMarkovByGuildId(guildId: string): Promise<Markov> {
+  const id = parseInt(guildId, 10);
+  const markov = new Markov({ id, options: markovOpts });
+  await markov.setup(); // Connect the markov instance to the DB to assign it an ID
+  return markov;
+}
+
+/**
+ * Checks if the author of a message as moderator-like permissions.
+ * @param {GuildMember} member Sender of the message
+ * @return {Boolean} True if the sender is a moderator.
+ * #v3-complete
+ */
+function isModerator(member: Discord.GuildMember | APIInteractionGuildMember | null): boolean {
+  const MODERATOR_PERMISSIONS: Discord.PermissionResolvable[] = [
+    'ADMINISTRATOR',
+    'MANAGE_CHANNELS',
+    'KICK_MEMBERS',
+    'MOVE_MEMBERS',
+  ];
+  if (!member) return false;
+  if (member instanceof Discord.GuildMember) {
+    return (
+      MODERATOR_PERMISSIONS.some((p) => member.permissions.has(p)) ||
+      config.ownerIds.includes(member.id)
+    );
+  }
+  // TODO: How to parse API permissions?
+  L.debug({ permissions: member.permissions });
+  return true;
+}
+
+type MessageCommands = 'respond' | 'train' | 'help' | 'invite' | 'debug' | 'tts' | null;
+
+/**
+ * Reads a new message and checks if and which command it is.
+ * @param {Message} message Message to be interpreted as a command
+ * @return {String} Command string
+ */
+function validateMessage(message: Discord.Message): MessageCommands {
+  const messageText = message.content.toLowerCase();
+  let command: MessageCommands = null;
+  const thisPrefix = messageText.substring(0, config.commandPrefix.length);
+  if (thisPrefix === config.commandPrefix) {
+    const split = messageText.split(' ');
+    if (split[0] === config.commandPrefix && split.length === 1) {
+      command = 'respond';
+    } else if (split[1] === 'train') {
+      command = 'train';
+    } else if (split[1] === 'help') {
+      command = 'help';
+    } else if (split[1] === 'invite') {
+      command = 'invite';
+    } else if (split[1] === 'debug') {
+      command = 'debug';
+    } else if (split[1] === 'tts') {
+      command = 'tts';
+    }
+  }
+  return command;
+}
+
+function messageToData(message: Discord.Message): AddDataProps {
+  const attachmentUrls = message.attachments.map((a) => a.url);
+  let custom: MarkovDataCustom | undefined;
+  if (attachmentUrls.length) custom = { attachments: attachmentUrls };
+  return {
+    string: message.content,
+    custom,
+  };
+}
+
+/**
+ * Recursively gets all messages in a text channel's history.
+ * #v3-complete
+ */
+async function saveChannelMessageHistory(
+  channel: Discord.TextChannel,
+  interaction: Discord.Message | Discord.CommandInteraction
+): Promise<void> {
+  if (!isModerator(interaction.member as any)) return;
+  const markov = await getMarkovByGuildId(channel.guildId);
+  L.debug({ channelId: channel.id }, `Training from text channel`);
+  const PAGE_SIZE = 100;
+  let keepGoing = true;
+  let oldestMessageID: string | undefined;
+
+  let channelMessagesCount = 0;
+
+  while (keepGoing) {
+    // eslint-disable-next-line no-await-in-loop
+    const messages = await channel.messages.fetch({
+      before: oldestMessageID,
+      limit: PAGE_SIZE,
+    });
+    const nonBotMessageFormatted = messages.filter((elem) => !elem.author.bot).map(messageToData);
+    L.debug({ oldestMessageID }, `Saving ${nonBotMessageFormatted.length} messages`);
+    // eslint-disable-next-line no-await-in-loop
+    await markov.addData(nonBotMessageFormatted);
+    L.trace('Finished saving messages');
+    channelMessagesCount += nonBotMessageFormatted.length;
+    const lastMessage = messages.last();
+    if (!lastMessage || messages.size < PAGE_SIZE) {
+      keepGoing = false;
+    } else {
+      oldestMessageID = lastMessage.id;
+    }
+  }
+
+  L.info(
+    { channelId: channel.id },
+    `Trained from ${channelMessagesCount} past human authored messages.`
+  );
+  await interaction.reply(`Trained from ${channelMessagesCount} past human authored messages.`);
+}
+
+/**
+ * General Markov-chain response function
+ * @param interaction The message that invoked the action, used for channel info.
+ * @param debug Sends debug info as a message if true.
+ * @param tts If the message should be sent as TTS. Defaults to the TTS setting of the
+ * invoking message.
+ * #v3-complete
+ */
+async function generateResponse(
+  interaction: Discord.Message | Discord.CommandInteraction,
+  debug = false,
+  tts = false
+): Promise<void> {
+  L.debug('Responding...');
+  if (!interaction.guildId) {
+    L.info('Received a message without a guildId');
+    return;
+  }
+  const options: MarkovGenerateOptions<MarkovDataCustom> = {
+    filter: (result): boolean => {
+      return result.score >= config.minScore;
+    },
+    maxTries: config.maxTries,
+  };
+
+  const markov = await getMarkovByGuildId(interaction.guildId);
+
+  try {
+    const response = await markov.generate<MarkovDataCustom>(options);
+    L.info({ response }, 'Generated response');
+    const messageOpts: Discord.MessageOptions = { tts };
+    const attachmentUrls = response.refs
+      .filter((ref) => ref.custom && 'attachments' in ref.custom)
+      .flatMap((ref) => ref.custom.attachments);
+    if (attachmentUrls.length > 0) {
+      const randomRefAttachment = getRandomElement(attachmentUrls);
+      messageOpts.files = [randomRefAttachment];
+    } else {
+      // TODO: This might not even work
+      const randomMessage = await MarkovInputData.createQueryBuilder<
+        MarkovInputData<MarkovDataCustom>
+      >('input')
+        .leftJoinAndSelect('input.fragment', 'fragment')
+        .leftJoinAndSelect('fragment.corpusEntry', 'corpusEntry')
+        .where([
+          {
+            fragment: { startWordMarkov: markov.db },
+          },
+          {
+            fragment: { endWordMarkov: markov.db },
+          },
+          {
+            fragment: { corpusEntry: { markov: markov.db } },
+          },
+        ])
+        .orderBy('RANDOM()')
+        .limit(1)
+        .getOne();
+      const randomMessageAttachmentUrls = randomMessage?.custom?.attachments;
+      if (randomMessageAttachmentUrls?.length) {
+        messageOpts.files = [{ attachment: getRandomElement(randomMessageAttachmentUrls) }];
+      }
+    }
+
+    response.string = response.string.replace(/@everyone/g, '@everyοne'); // Replace @everyone with a homoglyph 'o'
+    messageOpts.content = response.string;
+
+    if (interaction instanceof Discord.Message) {
+      await interaction.channel.send(messageOpts);
+      if (debug) {
+        await interaction.channel.send(`\`\`\`\n${JSON.stringify(response, null, 2)}\n\`\`\``);
+      }
+    } else if (interaction instanceof Discord.CommandInteraction) {
+      await interaction.editReply(messageOpts);
+      if (debug) {
+        await interaction.followUp(`\`\`\`\n${JSON.stringify(response, null, 2)}\n\`\`\``);
+      }
+    }
+  } catch (err) {
+    L.error(err);
+    if (debug) {
+      if (interaction instanceof Discord.Message) {
+        await interaction.channel.send(`\n\`\`\`\nERROR: ${err}\n\`\`\``);
+      } else if (interaction instanceof Discord.CommandInteraction) {
+        await interaction.editReply(`\n\`\`\`\nERROR: ${err}\n\`\`\``);
+      }
+    }
+  }
+}
+
+function helpMessage(): Discord.MessageOptions {
+  const avatarURL = client.user.avatarURL() || undefined;
+  const embed = new Discord.MessageEmbed()
+    .setAuthor(client.user.username || packageJson().name, avatarURL)
+    .setThumbnail(avatarURL as string)
+    .setDescription('A Markov chain chatbot that speaks based on previous chat input.')
+    .addField(
+      `${config.commandPrefix}`,
+      'Generates a sentence to say based on the chat database. Send your ' +
+        'message as TTS to recieve it as TTS.'
+    )
+    .addField(
+      `${config.commandPrefix} train`,
+      'Fetches the maximum amount of previous messages in the current ' +
+        'text channel, adds it to the database, and regenerates the corpus. Takes some time.'
+    )
+    .addField(
+      `${config.commandPrefix} invite`,
+      "Don't invite this bot to other servers. The database is shared " +
+        'between all servers and text channels.'
+    )
+    .addField(
+      `${config.commandPrefix} debug`,
+      `Runs the ${config.commandPrefix} command and follows it up with debug info.`
+    )
+    .setFooter(`Markov Discord ${getVersion()} by ${packageJson().author}`);
+  return {
+    embeds: [embed],
+  };
+}
+
+function inviteMessage(): Discord.MessageOptions {
+  const avatarURL = client.user.avatarURL() || undefined;
+  const embed = new Discord.MessageEmbed()
+    .setAuthor(`Invite ${client.user?.username}`, avatarURL)
+    .setThumbnail(avatarURL as string)
+    .addField(
+      'Invite',
+      `[Invite ${client.user.username} to your server](https://discord.com/api/oauth2/authorize?client_id=${client.user.id}&permissions=105472&scope=bot%20applications.commands)`
+    );
+  return { embeds: [embed] };
+}
+
+client.on('ready', async (readyClient) => {
+  L.info('Bot logged in');
+
+  await deployCommands(readyClient.user.id);
+
+  const guildsToSave: Guild[] = [];
+  const channelsToSave: Channel[] = [];
+  readyClient.guilds.valueOf().forEach((guild) => {
+    const dbGuild = Guild.create({ id: guild.id });
+    const textChannels = guild.channels.valueOf().filter((channel) => channel.isText());
+    const dbChannels = textChannels.map((channel) =>
+      Channel.create({ id: channel.id, guild: dbGuild })
+    );
+    guildsToSave.push(dbGuild);
+    channelsToSave.push(...dbChannels);
+  });
+  await Guild.upsert(guildsToSave, ['id']);
+  await Channel.upsert(channelsToSave, ['id']); // TODO: ensure this doesn't overwrite the existing `listen`
+});
+
+client.on('error', (err) => {
+  L.error(err);
+});
+
+client.on('messageCreate', async (message) => {
+  if (!(message.guild && message.channel instanceof Discord.TextChannel)) return;
+  const command = validateMessage(message);
+  if (command === 'help') {
+    await message.channel.send(helpMessage());
+  }
+  if (command === 'invite') {
+    await message.channel.send(inviteMessage());
+  }
+  if (command === 'train') {
+    await saveChannelMessageHistory(message.channel, message);
+  }
+  if (command === 'respond') {
+    await generateResponse(message);
+  }
+  if (command === 'tts') {
+    await generateResponse(message, false, true);
+  }
+  if (command === 'debug') {
+    await generateResponse(message, true);
+  }
+  if (command === null) {
+    L.debug('Listening...');
+    if (!message.author.bot) {
+      const markov = await getMarkovByGuildId(message.channel.guildId);
+      await markov.addData([messageToData(message)]);
+
+      if (client.user && message.mentions.has(client.user)) {
+        await generateResponse(message);
+      }
+    }
+  }
+});
+
+/**
+ * #v3-complete
+ */
+client.on('messageDelete', async (message) => {
+  L.info(`Deleting message ${message.id}`);
+  if (!(message.guildId && message.content)) {
+    return;
+  }
+  const markov = await getMarkovByGuildId(message.guildId);
+  await markov.removeData([message.content]);
+});
+
+/**
+ * #v3-complete
+ */
+client.on('messageUpdate', async (oldMessage, newMessage) => {
+  L.info(`Editing message ${oldMessage.id}`);
+  if (!(oldMessage.guildId && oldMessage.content && newMessage.content)) {
+    return;
+  }
+  const markov = await getMarkovByGuildId(oldMessage.guildId);
+  await markov.removeData([oldMessage.content]);
+  await markov.addData([newMessage.content]);
+});
+
+/**
+ * Loads the config settings from disk
+ */
+async function main(): Promise<void> {
+  const connection = await Markov.extendConnectionOptions();
+  await createConnection(connection);
+  await client.login(config.token);
+
+  // Move config if in legacy location
+  // TODO: import legacy DB?
+}
+
+main();

src/logger.ts

@@ -0,0 +1,22 @@
+import pino from 'pino';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const logger = pino({
+  transport: {
+    target: 'pino-pretty',
+    options: {
+      translateTime: `SYS:standard`,
+    },
+  },
+  formatters: {
+    level: (label) => {
+      return { level: label };
+    },
+  },
+  level: process.env.LOG_LEVEL || 'info',
+  base: undefined,
+});
+
+export default logger;

src/util.ts

@@ -0,0 +1,21 @@
+import fs from 'fs-extra';
+import path from 'path';
+import type { PackageJson } from 'types-package-json';
+
+let packageJsonCache: PackageJson | undefined;
+export const packageJson = (): PackageJson => {
+  if (packageJsonCache) return packageJsonCache;
+  packageJsonCache = fs.readJSONSync(path.resolve(process.cwd(), `package.json`));
+  return packageJsonCache as PackageJson;
+};
+
+export const getVersion = (): string => {
+  const { COMMIT_SHA } = process.env;
+  let { version } = packageJson();
+  if (COMMIT_SHA) version = `${version}#${COMMIT_SHA.substring(0, 8)}`;
+  return version;
+};
+
+export const getRandomElement = <T>(array: T[]): T => {
+  return array[Math.floor(Math.random() * array.length)];
+};