From d98d0fcd48ffe3c2fcde2114600ffc2faaed52ad Mon Sep 17 00:00:00 2001
From: Saoud Rizwan <7799382+saoudrizwan@users.noreply.github.com>
Date: Fri, 30 Aug 2024 06:21:34 -0400
Subject: [PATCH] Globby level by level and truncate at 1000 results to keep
 recursive list_files efficient

---
 src/ClaudeDev.ts               |  9 ++++----
 src/parse-source-code/index.ts | 38 ++++++++++++++++++++++++++++++++--
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/src/ClaudeDev.ts b/src/ClaudeDev.ts
index d779a05..ab0c5b1 100644
--- a/src/ClaudeDev.ts
+++ b/src/ClaudeDev.ts
@@ -12,7 +12,7 @@ import { serializeError } from "serialize-error"
 import treeKill from "tree-kill"
 import * as vscode from "vscode"
 import { ApiHandler, buildApiHandler } from "./api"
-import { listFiles, parseSourceCodeForDefinitionsTopLevel } from "./parse-source-code"
+import { LIST_FILES_LIMIT, listFiles, parseSourceCodeForDefinitionsTopLevel } from "./parse-source-code"
 import { ClaudeDevProvider } from "./providers/ClaudeDevProvider"
 import { ApiConfiguration } from "./shared/api"
 import { ClaudeRequestResult } from "./shared/ClaudeRequestResult"
@@ -1016,10 +1016,9 @@ export class ClaudeDev {
 				return a.localeCompare(b, undefined, { numeric: true, sensitivity: "base" })
 			})
 
-		if (sorted.length > 1000) {
-			const truncatedList = sorted.slice(0, 1000).join("\n")
-			const remainingCount = sorted.length - 1000
-			return `${truncatedList}\n\n(${remainingCount} files not listed due to automatic truncation. Try listing files in subdirectories if you need to explore further.)`
+		if (sorted.length >= LIST_FILES_LIMIT) {
+			const truncatedList = sorted.slice(0, LIST_FILES_LIMIT).join("\n")
+			return `${truncatedList}\n\n(Truncated at ${LIST_FILES_LIMIT} results. Try listing files in subdirectories if you need to explore further.)`
 		} else if (sorted.length === 0 || (sorted.length === 1 && sorted[0] === "")) {
 			return "No files found or you do not have permission to view this directory."
 		} else {
diff --git a/src/parse-source-code/index.ts b/src/parse-source-code/index.ts
index 7d59b21..5a9d6f3 100644
--- a/src/parse-source-code/index.ts
+++ b/src/parse-source-code/index.ts
@@ -1,9 +1,11 @@
 import * as fs from "fs/promises"
-import { globby } from "globby"
+import { globby, Options } from "globby"
 import os from "os"
 import * as path from "path"
 import { LanguageParser, loadRequiredLanguageParsers } from "./languageParser"
 
+export const LIST_FILES_LIMIT = 1000
+
 // TODO: implement caching behavior to avoid having to keep analyzing project for new tasks.
 export async function parseSourceCodeForDefinitionsTopLevel(dirPath: string): Promise<string> {
 	// check if the path exists
@@ -96,10 +98,42 @@ export async function listFiles(dirPath: string, recursive: boolean): Promise<st
 		onlyFiles: false, // true by default, false means it will list directories on their own too
 	}
 	// * globs all files in one dir, ** globs files in nested directories
-	const files = await globby(recursive ? "**" : "*", options)
+	const files = recursive ? await globbyLevelByLevel(options) : await globby("*", options)
 	return files
 }
 
+// globby doesnt natively support top down level by level globbing, so we implement it ourselves
+async function globbyLevelByLevel(options?: Options) {
+	let results: string[] = []
+	let currentLevel = 1
+	while (results.length < LIST_FILES_LIMIT) {
+		// Construct the glob pattern for the current level
+		const pattern = `${"*/".repeat(currentLevel)}*`
+
+		// Get files and directories at the current level
+		const filesAtLevel = await globby(pattern, options)
+
+		// If no more files found at this level, break the loop
+		if (filesAtLevel.length === 0) {
+			break
+		}
+
+		// Add the files found at this level to the result
+		results.push(...filesAtLevel)
+
+		// If we have reached the max limit, slice the array to the limit and break
+		if (results.length >= LIST_FILES_LIMIT) {
+			results = results.slice(0, LIST_FILES_LIMIT)
+			break
+		}
+
+		// Move to the next level
+		currentLevel++
+	}
+
+	return results
+}
+
 function separateFiles(allFiles: string[]): { filesToParse: string[]; remainingFiles: string[] } {
 	const extensions = [
 		"js",