Track progress of training

package-lock.json

@@ -16,6 +16,7 @@
         "bufferutil": "^4.0.5",
         "class-transformer": "^0.5.1",
         "class-validator": "^0.13.2",
+        "date-fns": "^2.28.0",
         "discord-api-types": "^0.25.2",
         "discord.js": "^13.3.1",
         "dotenv": "^10.0.0",
@@ -26,6 +27,7 @@
         "pino": "^7.5.1",
         "pino-pretty": "^7.3.0",
         "reflect-metadata": "^0.1.13",
+        "simple-eta": "^3.0.2",
         "source-map-support": "^0.5.21",
         "typeorm": "^0.2.38",
         "utf-8-validate": "^5.0.7",
@@ -1122,6 +1124,18 @@
         "node": ">= 8"
       }
     },
+    "node_modules/date-fns": {
+      "version": "2.28.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.28.0.tgz",
+      "integrity": "sha512-8d35hViGYx/QH0icHYCeLmsLmMUheMmTyV9Fcm6gvNwdw31yXXH+O85sOBJ+OLnLQMKZowvpKb6FgMIQjcpvQw==",
+      "engines": {
+        "node": ">=0.11"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/date-fns"
+      }
+    },
     "node_modules/dateformat": {
       "version": "4.6.3",
       "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
@@ -3581,6 +3595,11 @@
         }
       ]
     },
+    "node_modules/simple-eta": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/simple-eta/-/simple-eta-3.0.2.tgz",
+      "integrity": "sha512-+OmPgi01yHK/bRNQDoehUcV8fqs9nNJkG2DoWCnnLvj0lmowab7BH3v9776BG0y7dGEOLh0F7mfd37k+ht26Yw=="
+    },
     "node_modules/simple-get": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz",
@@ -5253,6 +5272,11 @@
         "which": "^2.0.1"
       }
     },
+    "date-fns": {
+      "version": "2.28.0",
+      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.28.0.tgz",
+      "integrity": "sha512-8d35hViGYx/QH0icHYCeLmsLmMUheMmTyV9Fcm6gvNwdw31yXXH+O85sOBJ+OLnLQMKZowvpKb6FgMIQjcpvQw=="
+    },
     "dateformat": {
       "version": "4.6.3",
       "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
@@ -7074,6 +7098,11 @@
       "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
       "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q=="
     },
+    "simple-eta": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/simple-eta/-/simple-eta-3.0.2.tgz",
+      "integrity": "sha512-+OmPgi01yHK/bRNQDoehUcV8fqs9nNJkG2DoWCnnLvj0lmowab7BH3v9776BG0y7dGEOLh0F7mfd37k+ht26Yw=="
+    },
     "simple-get": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.0.tgz",

package.json

@@ -37,6 +37,7 @@
     "bufferutil": "^4.0.5",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.13.2",
+    "date-fns": "^2.28.0",
     "discord-api-types": "^0.25.2",
     "discord.js": "^13.3.1",
     "dotenv": "^10.0.0",
@@ -47,6 +48,7 @@
     "pino": "^7.5.1",
     "pino-pretty": "^7.3.0",
     "reflect-metadata": "^0.1.13",
+    "simple-eta": "^3.0.2",
     "source-map-support": "^0.5.21",
     "typeorm": "^0.2.38",
     "utf-8-validate": "^5.0.7",

src/index.ts

@@ -16,6 +16,9 @@ import {
   APISelectMenuComponent,
   APIInteractionGuildMember,
 } from 'discord.js/node_modules/discord-api-types';
+import makeEta from 'simple-eta';
+import formatDistanceToNow from 'date-fns/formatDistanceToNow';
+import addSeconds from 'date-fns/addSeconds';
 import L from './logger';
 import { Channel } from './entity/Channel';
 import { Guild } from './entity/Guild';
@@ -209,13 +212,53 @@ async function saveGuildMessageHistory(
   const channelIds = channels.map((c) => c.id);
   L.debug({ channelIds }, `Training from text channels`);
 
+  const messageContent = `Parsing past messages from ${channels.length} channel(s).`;
+
+  const completedChannelsField: Discord.EmbedFieldData = {
+    name: 'Completed Channels',
+    value: 'None',
+    inline: true,
+  };
+  const currentChannelField: Discord.EmbedFieldData = {
+    name: 'Current Channel',
+    value: `<#${channels[0].id}>`,
+    inline: true,
+  };
+  const currentChannelPercent: Discord.EmbedFieldData = {
+    name: 'Channel Progress',
+    value: '0%',
+    inline: true,
+  };
+  const currentChannelEta: Discord.EmbedFieldData = {
+    name: 'Channel Time Remaining',
+    value: 'Pending...',
+    inline: true,
+  };
+  const embedOptions: Discord.MessageEmbedOptions = {
+    title: 'Training Progress',
+    fields: [completedChannelsField, currentChannelField, currentChannelPercent, currentChannelEta],
+  };
+  const embed = new Discord.MessageEmbed(embedOptions);
+  let progressMessage: Discord.Message;
+  const updateMessageData = { content: messageContent, embeds: [embed] };
+  if (interaction instanceof Discord.Message) {
+    progressMessage = await interaction.reply(updateMessageData);
+  } else {
+    progressMessage = (await interaction.followUp(updateMessageData)) as Discord.Message;
+  }
+
   const PAGE_SIZE = 100;
+  const UPDATE_RATE = 1000; // In number of messages processed
+  let lastUpdate = 0;
   let messagesCount = 0;
+  let firstMessageDate: number | undefined;
   // eslint-disable-next-line no-restricted-syntax
   for (const channel of channels) {
     let oldestMessageID: string | undefined;
     let keepGoing = true;
     L.debug({ channelId: channel.id, messagesCount }, `Training from channel`);
+    const channelCreateDate = channel.createdTimestamp;
+    const channelEta = makeEta({ autostart: true, min: 0, max: 1, historyTimeConstant: 10 });
 
     while (keepGoing) {
       // eslint-disable-next-line no-await-in-loop
@@ -230,11 +273,43 @@ async function saveGuildMessageHistory(
       L.trace('Finished saving messages');
       messagesCount += nonBotMessageFormatted.length;
       const lastMessage = messages.last();
+
+      // Update tracking metrics
       if (!lastMessage || messages.size < PAGE_SIZE) {
         keepGoing = false;
+        if (completedChannelsField.value === 'None') completedChannelsField.value = '';
+        completedChannelsField.value += `\n • <#${channel.id}>`;
       } else {
         oldestMessageID = lastMessage.id;
       }
+      currentChannelField.value = `<#${channel.id}>`;
+      if (!firstMessageDate) firstMessageDate = messages.first()?.createdTimestamp;
+      const oldestMessageDate = lastMessage?.createdTimestamp;
+      if (firstMessageDate && oldestMessageDate) {
+        const channelAge = firstMessageDate - channelCreateDate;
+        const lastMessageAge = firstMessageDate - oldestMessageDate;
+        const pctComplete = lastMessageAge / channelAge;
+        currentChannelPercent.value = `${pctComplete.toFixed(2)}%`;
+        channelEta.report(pctComplete);
+        const estimateSeconds = channelEta.estimate();
+        if (Number.isFinite(estimateSeconds))
+          currentChannelEta.value = formatDistanceToNow(addSeconds(new Date(), estimateSeconds), {
+            includeSeconds: true,
+          });
+      }
+
+      if (messagesCount > lastUpdate + UPDATE_RATE) {
+        lastUpdate = messagesCount;
+        L.debug(
+          { messagesCount, pctComplete: currentChannelPercent.value },
+          'Sending metrics update'
+        );
+        // eslint-disable-next-line no-await-in-loop
+        await progressMessage.edit({
+          ...updateMessageData,
+          embeds: [new Discord.MessageEmbed(embedOptions)],
+        });
+      }
     }
   }