From 705cea80e7718c9acdcbf95b171dd6b69c0d8d27 Mon Sep 17 00:00:00 2001 From: Charlie Laabs Date: Thu, 30 Dec 2021 17:19:57 -0600 Subject: [PATCH] Track progress of training --- package-lock.json | 29 ++++++++++++++++++ package.json | 2 ++ src/index.ts | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/package-lock.json b/package-lock.json index 370665a..098a028 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "bufferutil": "^4.0.5", "class-transformer": "^0.5.1", "class-validator": "^0.13.2", + "date-fns": "^2.28.0", "discord-api-types": "^0.25.2", "discord.js": "^13.3.1", "dotenv": "^10.0.0", @@ -26,6 +27,7 @@ "pino": "^7.5.1", "pino-pretty": "^7.3.0", "reflect-metadata": "^0.1.13", + "simple-eta": "^3.0.2", "source-map-support": "^0.5.21", "typeorm": "^0.2.38", "utf-8-validate": "^5.0.7", @@ -1122,6 +1124,18 @@ "node": ">= 8" } }, + "node_modules/date-fns": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.28.0.tgz", + "integrity": "sha512-8d35hViGYx/QH0icHYCeLmsLmMUheMmTyV9Fcm6gvNwdw31yXXH+O85sOBJ+OLnLQMKZowvpKb6FgMIQjcpvQw==", + "engines": { + "node": ">=0.11" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/date-fns" + } + }, "node_modules/dateformat": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", @@ -3581,6 +3595,11 @@ } ] }, + "node_modules/simple-eta": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/simple-eta/-/simple-eta-3.0.2.tgz", + "integrity": "sha512-+OmPgi01yHK/bRNQDoehUcV8fqs9nNJkG2DoWCnnLvj0lmowab7BH3v9776BG0y7dGEOLh0F7mfd37k+ht26Yw==" + }, "node_modules/simple-get": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz", @@ -5253,6 +5272,11 @@ "which": "^2.0.1" } }, + "date-fns": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.28.0.tgz", + "integrity": "sha512-8d35hViGYx/QH0icHYCeLmsLmMUheMmTyV9Fcm6gvNwdw31yXXH+O85sOBJ+OLnLQMKZowvpKb6FgMIQjcpvQw==" + }, "dateformat": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", @@ -7074,6 +7098,11 @@ "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" }, + "simple-eta": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/simple-eta/-/simple-eta-3.0.2.tgz", + "integrity": "sha512-+OmPgi01yHK/bRNQDoehUcV8fqs9nNJkG2DoWCnnLvj0lmowab7BH3v9776BG0y7dGEOLh0F7mfd37k+ht26Yw==" + }, "simple-get": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz", diff --git a/package.json b/package.json index f95a710..3f22152 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "bufferutil": "^4.0.5", "class-transformer": "^0.5.1", "class-validator": "^0.13.2", + "date-fns": "^2.28.0", "discord-api-types": "^0.25.2", "discord.js": "^13.3.1", "dotenv": "^10.0.0", @@ -47,6 +48,7 @@ "pino": "^7.5.1", "pino-pretty": "^7.3.0", "reflect-metadata": "^0.1.13", + "simple-eta": "^3.0.2", "source-map-support": "^0.5.21", "typeorm": "^0.2.38", "utf-8-validate": "^5.0.7", diff --git a/src/index.ts b/src/index.ts index 1fc7df5..ded0ebb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,9 @@ import { APISelectMenuComponent, APIInteractionGuildMember, } from 'discord.js/node_modules/discord-api-types'; +import makeEta from 'simple-eta'; +import formatDistanceToNow from 'date-fns/formatDistanceToNow'; +import addSeconds from 'date-fns/addSeconds'; import L from './logger'; import { Channel } from './entity/Channel'; import { Guild } from './entity/Guild'; @@ -209,13 +212,53 @@ async function saveGuildMessageHistory( const channelIds = channels.map((c) => c.id); L.debug({ channelIds }, `Training from text channels`); + const messageContent = `Parsing past messages from ${channels.length} channel(s).`; + + const completedChannelsField: Discord.EmbedFieldData = { + name: 'Completed Channels', + value: 'None', + inline: true, + }; + const currentChannelField: Discord.EmbedFieldData = { + name: 'Current Channel', + value: `<#${channels[0].id}>`, + inline: true, + }; + const currentChannelPercent: Discord.EmbedFieldData = { + name: 'Channel Progress', + value: '0%', + inline: true, + }; + const currentChannelEta: Discord.EmbedFieldData = { + name: 'Channel Time Remaining', + value: 'Pending...', + inline: true, + }; + const embedOptions: Discord.MessageEmbedOptions = { + title: 'Training Progress', + fields: [completedChannelsField, currentChannelField, currentChannelPercent, currentChannelEta], + }; + const embed = new Discord.MessageEmbed(embedOptions); + let progressMessage: Discord.Message; + const updateMessageData = { content: messageContent, embeds: [embed] }; + if (interaction instanceof Discord.Message) { + progressMessage = await interaction.reply(updateMessageData); + } else { + progressMessage = (await interaction.followUp(updateMessageData)) as Discord.Message; + } + const PAGE_SIZE = 100; + const UPDATE_RATE = 1000; // In number of messages processed + let lastUpdate = 0; let messagesCount = 0; + let firstMessageDate: number | undefined; // eslint-disable-next-line no-restricted-syntax for (const channel of channels) { let oldestMessageID: string | undefined; let keepGoing = true; L.debug({ channelId: channel.id, messagesCount }, `Training from channel`); + const channelCreateDate = channel.createdTimestamp; + const channelEta = makeEta({ autostart: true, min: 0, max: 1, historyTimeConstant: 10 }); while (keepGoing) { // eslint-disable-next-line no-await-in-loop @@ -230,11 +273,43 @@ async function saveGuildMessageHistory( L.trace('Finished saving messages'); messagesCount += nonBotMessageFormatted.length; const lastMessage = messages.last(); + + // Update tracking metrics if (!lastMessage || messages.size < PAGE_SIZE) { keepGoing = false; + if (completedChannelsField.value === 'None') completedChannelsField.value = ''; + completedChannelsField.value += `\n • <#${channel.id}>`; } else { oldestMessageID = lastMessage.id; } + currentChannelField.value = `<#${channel.id}>`; + if (!firstMessageDate) firstMessageDate = messages.first()?.createdTimestamp; + const oldestMessageDate = lastMessage?.createdTimestamp; + if (firstMessageDate && oldestMessageDate) { + const channelAge = firstMessageDate - channelCreateDate; + const lastMessageAge = firstMessageDate - oldestMessageDate; + const pctComplete = lastMessageAge / channelAge; + currentChannelPercent.value = `${pctComplete.toFixed(2)}%`; + channelEta.report(pctComplete); + const estimateSeconds = channelEta.estimate(); + if (Number.isFinite(estimateSeconds)) + currentChannelEta.value = formatDistanceToNow(addSeconds(new Date(), estimateSeconds), { + includeSeconds: true, + }); + } + + if (messagesCount > lastUpdate + UPDATE_RATE) { + lastUpdate = messagesCount; + L.debug( + { messagesCount, pctComplete: currentChannelPercent.value }, + 'Sending metrics update' + ); + // eslint-disable-next-line no-await-in-loop + await progressMessage.edit({ + ...updateMessageData, + embeds: [new Discord.MessageEmbed(embedOptions)], + }); + } } }