feat: Implement optimization plan for Markov Discord bot

- Added `optimization-plan.md` detailing strategies to reduce response latency and improve training throughput.
- Enhanced performance analysis in `performance-analysis.md` with identified bottlenecks and completed optimizations.
- Created `productContext.md` summarizing project goals, user scenarios, and implementation priorities.
- Developed `markov-store.ts` for high-performance serialized chain storage with alias method sampling.
- Implemented database performance indexes in `1704067200000-AddPerformanceIndexes.ts`.
- Introduced `markov-worker.ts` for handling CPU-intensive operations in separate threads.
- Established a worker pool in `worker-pool.ts` to manage multiple worker threads efficiently.

package-lock.json

@@ -9,6 +9,7 @@
       "version": "2.3.0",
       "license": "MIT",
       "dependencies": {
+        "@types/stream-json": "^1.7.8",
         "better-sqlite3": "^9.6.0",
         "bufferutil": "^4.0.8",
         "class-transformer": "^0.5.1",
@@ -26,6 +27,7 @@
         "reflect-metadata": "^0.2.2",
         "simple-eta": "^3.0.2",
         "source-map-support": "^0.5.21",
+        "stream-json": "^1.9.1",
         "typeorm": "^0.3.20",
         "utf-8-validate": "^6.0.4",
         "zlib-sync": "^0.1.9"
@@ -663,22 +665,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/@pm2/agent/node_modules/utf-8-validate": {
-      "version": "5.0.10",
-      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
-      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
-      "dev": true,
-      "hasInstallScript": true,
-      "license": "MIT",
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "node-gyp-build": "^4.3.0"
-      },
-      "engines": {
-        "node": ">=6.14.2"
-      }
-    },
     "node_modules/@pm2/agent/node_modules/ws": {
       "version": "7.5.10",
       "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz",
@@ -851,22 +837,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/@pm2/js-api/node_modules/utf-8-validate": {
-      "version": "5.0.10",
-      "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz",
-      "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==",
-      "dev": true,
-      "hasInstallScript": true,
-      "license": "MIT",
-      "optional": true,
-      "peer": true,
-      "dependencies": {
-        "node-gyp-build": "^4.3.0"
-      },
-      "engines": {
-        "node": ">=6.14.2"
-      }
-    },
     "node_modules/@pm2/js-api/node_modules/ws": {
       "version": "7.5.10",
       "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.10.tgz",
@@ -1093,6 +1063,25 @@
         "undici-types": "~6.19.2"
       }
     },
+    "node_modules/@types/stream-chain": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@types/stream-chain/-/stream-chain-2.1.0.tgz",
+      "integrity": "sha512-guDyAl6s/CAzXUOWpGK2bHvdiopLIwpGu8v10+lb9hnQOyo4oj/ZUQFOvqFjKGsE3wJP1fpIesCcMvbXuWsqOg==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
+    "node_modules/@types/stream-json": {
+      "version": "1.7.8",
+      "resolved": "https://registry.npmjs.org/@types/stream-json/-/stream-json-1.7.8.tgz",
+      "integrity": "sha512-MU1OB1eFLcYWd1LjwKXrxdoPtXSRzRmAnnxs4Js/ayB5O/NvHraWwuOaqMWIebpYwM6khFlsJOHEhI9xK/ab4Q==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*",
+        "@types/stream-chain": "*"
+      }
+    },
     "node_modules/@types/unist": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
@@ -7381,6 +7370,21 @@
         "node": "^18.17.0 || >=20.5.0"
       }
     },
+    "node_modules/stream-chain": {
+      "version": "2.2.5",
+      "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz",
+      "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==",
+      "license": "BSD-3-Clause"
+    },
+    "node_modules/stream-json": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz",
+      "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "stream-chain": "^2.2.5"
+      }
+    },
     "node_modules/stream-shift": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz",