Revert "Remove web-tree-sitter"

This reverts commit 5c98a3d4af.

# Conflicts:
#	package.json

esbuild.js

@@ -66,7 +66,7 @@ const extensionConfig = {
 	sourcemap: !production,
 	logLevel: "silent",
 	plugins: [
-		//copyWasmFiles,
+		copyWasmFiles,
 		/* add to the end of plugins array */
 		esbuildProblemMatcherPlugin,
 	],

package-lock.json

@@ -17,7 +17,9 @@
         "globby": "^14.0.2",
         "os-name": "^6.0.0",
         "p-wait-for": "^5.0.2",
-        "serialize-error": "^11.0.3"
+        "serialize-error": "^11.0.3",
+        "tree-sitter-wasms": "^0.1.11",
+        "web-tree-sitter": "^0.22.6"
       },
       "devDependencies": {
         "@types/diff": "^5.2.1",
@@ -5614,6 +5616,11 @@
       "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
       "license": "MIT"
     },
+    "node_modules/tree-sitter-wasms": {
+      "version": "0.1.11",
+      "resolved": "https://registry.npmjs.org/tree-sitter-wasms/-/tree-sitter-wasms-0.1.11.tgz",
+      "integrity": "sha512-26sE4+qoTi1CbzHdo9sHs9pRE/jXVFVRigSG/5TNAbwhSMVjHfMAg4UjmOhAFAIx5UxgoQuaURwqhm0SRNrpWA=="
+    },
     "node_modules/ts-api-utils": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz",
@@ -5829,6 +5836,11 @@
         "node": ">= 8"
       }
     },
+    "node_modules/web-tree-sitter": {
+      "version": "0.22.6",
+      "resolved": "https://registry.npmjs.org/web-tree-sitter/-/web-tree-sitter-0.22.6.tgz",
+      "integrity": "sha512-hS87TH71Zd6mGAmYCvlgxeGDjqd9GTeqXNqTT+u0Gs51uIozNIaaq/kUAbV/Zf56jb2ZOyG8BxZs2GG9wbLi6Q=="
+    },
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",

package.json

@@ -2,7 +2,7 @@
   "name": "claude-dev",
   "displayName": "Claude Dev",
   "description": "Autonomous software engineer right in your IDE, capable of reading/writing files, executing commands, and more with your permission every step of the way.",
-  "version": "1.0.64",
+  "version": "1.0.62",
   "icon": "icon.png",
   "engines": {
     "vscode": "^1.84.0"
@@ -130,6 +130,8 @@
     "globby": "^14.0.2",
     "os-name": "^6.0.0",
     "p-wait-for": "^5.0.2",
-    "serialize-error": "^11.0.3"
+    "serialize-error": "^11.0.3",
+    "tree-sitter-wasms": "^0.1.11",
+    "web-tree-sitter": "^0.22.6"
   }
 }

src/analyze-project/index.ts

@@ -1,7 +1,7 @@
 import * as fs from "fs/promises"
 import { globby } from "globby"
 import * as path from "path"
-//import { LanguageParser, loadRequiredLanguageParsers } from "./languageParser"
+import { LanguageParser, loadRequiredLanguageParsers } from "./languageParser"
 
 async function analyzeProject(dirPath: string): Promise<string> {
 	let result = ""
@@ -13,12 +13,12 @@ async function analyzeProject(dirPath: string): Promise<string> {
 	const { filesToParse, remainingFiles } = separateFiles(allFiles)
 
 	// Load only the necessary language parsers
-	//const languageParsers = await loadRequiredLanguageParsers(filesToParse)
+	const languageParsers = await loadRequiredLanguageParsers(filesToParse)
 
 	// Parse specific files we have language parsers for
 	const filesWithoutDefinitions: string[] = []
 	for (const file of filesToParse) {
-		const definitions = undefined //await parseFile(file, languageParsers)
+		const definitions = await parseFile(file, languageParsers)
 		if (definitions) {
 			if (!result) {
 				result += "# Source code definitions:\n\n"
@@ -119,68 +119,68 @@ This approach allows us to focus on the most relevant parts of the code (defined
 - https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/helper.js
 - https://tree-sitter.github.io/tree-sitter/code-navigation-systems
 */
-// async function parseFile(filePath: string, languageParsers: LanguageParser): Promise<string | undefined> {
+async function parseFile(filePath: string, languageParsers: LanguageParser): Promise<string | undefined> {
-// 	const fileContent = await fs.readFile(filePath, "utf8")
+	const fileContent = await fs.readFile(filePath, "utf8")
-// 	const ext = path.extname(filePath).toLowerCase().slice(1)
+	const ext = path.extname(filePath).toLowerCase().slice(1)
 
-// 	const { parser, query } = languageParsers[ext] || {}
+	const { parser, query } = languageParsers[ext] || {}
-// 	if (!parser || !query) {
+	if (!parser || !query) {
-// 		return `Unsupported file type: ${filePath}`
+		return `Unsupported file type: ${filePath}`
-// 	}
+	}
 
-// 	let formattedOutput = ""
+	let formattedOutput = ""
 
-// 	try {
+	try {
-// 		// Parse the file content into an Abstract Syntax Tree (AST), a tree-like representation of the code
+		// Parse the file content into an Abstract Syntax Tree (AST), a tree-like representation of the code
-// 		const tree = parser.parse(fileContent)
+		const tree = parser.parse(fileContent)
 
-// 		// Apply the query to the AST and get the captures
+		// Apply the query to the AST and get the captures
-// 		// Captures are specific parts of the AST that match our query patterns, each capture represents a node in the AST that we're interested in.
+		// Captures are specific parts of the AST that match our query patterns, each capture represents a node in the AST that we're interested in.
-// 		const captures = query.captures(tree.rootNode)
+		const captures = query.captures(tree.rootNode)
 
-// 		// Sort captures by their start position
+		// Sort captures by their start position
-// 		captures.sort((a, b) => a.node.startPosition.row - b.node.startPosition.row)
+		captures.sort((a, b) => a.node.startPosition.row - b.node.startPosition.row)
 
-// 		// Split the file content into individual lines
+		// Split the file content into individual lines
-// 		const lines = fileContent.split("\n")
+		const lines = fileContent.split("\n")
 
-// 		// Keep track of the last line we've processed
+		// Keep track of the last line we've processed
-// 		let lastLine = -1
+		let lastLine = -1
 
-// 		captures.forEach((capture) => {
+		captures.forEach((capture) => {
-// 			const { node, name } = capture
+			const { node, name } = capture
-// 			// Get the start and end lines of the current AST node
+			// Get the start and end lines of the current AST node
-// 			const startLine = node.startPosition.row
+			const startLine = node.startPosition.row
-// 			const endLine = node.endPosition.row
+			const endLine = node.endPosition.row
-// 			// Once we've retrieved the nodes we care about through the language query, we filter for lines with definition names only.
+			// Once we've retrieved the nodes we care about through the language query, we filter for lines with definition names only.
-// 			// name.startsWith("name.reference.") > refs can be used for ranking purposes, but we don't need them for the output
+			// name.startsWith("name.reference.") > refs can be used for ranking purposes, but we don't need them for the output
-// 			// previously we did `name.startsWith("name.definition.")` but this was too strict and excluded some relevant definitions
+			// previously we did `name.startsWith("name.definition.")` but this was too strict and excluded some relevant definitions
 
-// 			// Add separator if there's a gap between captures
+			// Add separator if there's a gap between captures
-// 			if (lastLine !== -1 && startLine > lastLine + 1) {
+			if (lastLine !== -1 && startLine > lastLine + 1) {
-// 				formattedOutput += "|----\n"
+				formattedOutput += "|----\n"
-// 			}
+			}
-// 			// Only add the first line of the definition
+			// Only add the first line of the definition
-// 			// query captures includes the definition name and the definition implementation, but we only want the name (I found discrepencies in the naming structure for various languages, i.e. javascript names would be 'name' and typescript names would be 'name.definition)
+			// query captures includes the definition name and the definition implementation, but we only want the name (I found discrepencies in the naming structure for various languages, i.e. javascript names would be 'name' and typescript names would be 'name.definition)
-// 			if (name.includes("name") && lines[startLine]) {
+			if (name.includes("name") && lines[startLine]) {
-// 				formattedOutput += `│${lines[startLine]}\n`
+				formattedOutput += `│${lines[startLine]}\n`
-// 			}
+			}
-// 			// Adds all the captured lines
+			// Adds all the captured lines
-// 			// for (let i = startLine; i <= endLine; i++) {
+			// for (let i = startLine; i <= endLine; i++) {
-// 			// 	formattedOutput += `│${lines[i]}\n`
+			// 	formattedOutput += `│${lines[i]}\n`
-// 			// }
+			// }
-// 			//}
+			//}
 
-// 			lastLine = endLine
+			lastLine = endLine
-// 		})
+		})
-// 	} catch (error) {
+	} catch (error) {
-// 		console.log(`Error parsing file: ${error}\n`)
+		console.log(`Error parsing file: ${error}\n`)
-// 	}
+	}
 
-// 	if (formattedOutput.length > 0) {
+	if (formattedOutput.length > 0) {
-// 		return `|----\n${formattedOutput}|----\n`
+		return `|----\n${formattedOutput}|----\n`
-// 	}
+	}
-// 	return undefined
+	return undefined
-// }
+}
 
 export { analyzeProject }

src/analyze-project/languageParser.ts

@@ -1,122 +1,122 @@
-// import * as path from "path"
+import * as path from "path"
-// import Parser from "web-tree-sitter"
+import Parser from "web-tree-sitter"
-// import {
+import {
-// 	javascriptQuery,
+	javascriptQuery,
-// 	typescriptQuery,
+	typescriptQuery,
-// 	pythonQuery,
+	pythonQuery,
-// 	rustQuery,
+	rustQuery,
-// 	goQuery,
+	goQuery,
-// 	cppQuery,
+	cppQuery,
-// 	cQuery,
+	cQuery,
-// 	csharpQuery,
+	csharpQuery,
-// 	rubyQuery,
+	rubyQuery,
-// 	javaQuery,
+	javaQuery,
-// 	phpQuery,
+	phpQuery,
-// 	swiftQuery,
+	swiftQuery,
-// } from "./queries"
+} from "./queries"
 
-// export interface LanguageParser {
+export interface LanguageParser {
-// 	[key: string]: {
+	[key: string]: {
-// 		parser: Parser
+		parser: Parser
-// 		query: Parser.Query
+		query: Parser.Query
-// 	}
+	}
-// }
+}
 
-// async function loadLanguage(langName: string) {
+async function loadLanguage(langName: string) {
-// 	return await Parser.Language.load(path.join(__dirname, `tree-sitter-${langName}.wasm`))
+	return await Parser.Language.load(path.join(__dirname, `tree-sitter-${langName}.wasm`))
-// }
+}
 
-// /*
+/*
-// Using node bindings for tree-sitter is problematic in vscode extensions
+Using node bindings for tree-sitter is problematic in vscode extensions 
-// because of incompatibility with electron. Going the .wasm route has the
+because of incompatibility with electron. Going the .wasm route has the 
-// advantage of not having to build for multiple architectures.
+advantage of not having to build for multiple architectures.
 
-// We use web-tree-sitter and tree-sitter-wasms which provides auto-updating prebuilt WASM binaries for tree-sitter's language parsers.
+We use web-tree-sitter and tree-sitter-wasms which provides auto-updating prebuilt WASM binaries for tree-sitter's language parsers.
 
-// This function loads WASM modules for relevant language parsers based on input files:
+This function loads WASM modules for relevant language parsers based on input files:
-// 1. Extracts unique file extensions
+1. Extracts unique file extensions
-// 2. Maps extensions to language names
+2. Maps extensions to language names
-// 3. Loads corresponding WASM files (containing grammar rules)
+3. Loads corresponding WASM files (containing grammar rules)
-// 4. Uses WASM modules to initialize tree-sitter parsers
+4. Uses WASM modules to initialize tree-sitter parsers
 
-// This approach optimizes performance by loading only necessary parsers once for all relevant files.
+This approach optimizes performance by loading only necessary parsers once for all relevant files.
 
-// Sources:
+Sources:
-// - https://github.com/tree-sitter/node-tree-sitter/issues/169
+- https://github.com/tree-sitter/node-tree-sitter/issues/169
-// - https://github.com/tree-sitter/node-tree-sitter/issues/168
+- https://github.com/tree-sitter/node-tree-sitter/issues/168
-// - https://github.com/Gregoor/tree-sitter-wasms/blob/main/README.md
+- https://github.com/Gregoor/tree-sitter-wasms/blob/main/README.md
-// - https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/README.md
+- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/README.md
-// - https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/query-test.js
+- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/query-test.js
-// */
+*/
-// export async function loadRequiredLanguageParsers(filesToParse: string[]): Promise<LanguageParser> {
+export async function loadRequiredLanguageParsers(filesToParse: string[]): Promise<LanguageParser> {
-// 	await Parser.init()
+	await Parser.init()
-// 	const extensionsToLoad = new Set(filesToParse.map((file) => path.extname(file).toLowerCase().slice(1)))
+	const extensionsToLoad = new Set(filesToParse.map((file) => path.extname(file).toLowerCase().slice(1)))
-// 	const parsers: LanguageParser = {}
+	const parsers: LanguageParser = {}
-// 	for (const ext of extensionsToLoad) {
+	for (const ext of extensionsToLoad) {
-// 		let language: Parser.Language
+		let language: Parser.Language
-// 		let query: Parser.Query
+		let query: Parser.Query
-// 		switch (ext) {
+		switch (ext) {
-// 			case "js":
+			case "js":
-// 			case "jsx":
+			case "jsx":
-// 				language = await loadLanguage("javascript")
+				language = await loadLanguage("javascript")
-// 				query = language.query(javascriptQuery)
+				query = language.query(javascriptQuery)
-// 				break
+				break
-// 			case "ts":
+			case "ts":
-// 				language = await loadLanguage("typescript")
+				language = await loadLanguage("typescript")
-// 				query = language.query(typescriptQuery)
+				query = language.query(typescriptQuery)
-// 				break
+				break
-// 			case "tsx":
+			case "tsx":
-// 				language = await loadLanguage("tsx")
+				language = await loadLanguage("tsx")
-// 				query = language.query(typescriptQuery)
+				query = language.query(typescriptQuery)
-// 				break
+				break
-// 			case "py":
+			case "py":
-// 				language = await loadLanguage("python")
+				language = await loadLanguage("python")
-// 				query = language.query(pythonQuery)
+				query = language.query(pythonQuery)
-// 				break
+				break
-// 			case "rs":
+			case "rs":
-// 				language = await loadLanguage("rust")
+				language = await loadLanguage("rust")
-// 				query = language.query(rustQuery)
+				query = language.query(rustQuery)
-// 				break
+				break
-// 			case "go":
+			case "go":
-// 				language = await loadLanguage("go")
+				language = await loadLanguage("go")
-// 				query = language.query(goQuery)
+				query = language.query(goQuery)
-// 				break
+				break
-// 			case "cpp":
+			case "cpp":
-// 			case "hpp":
+			case "hpp":
-// 				language = await loadLanguage("cpp")
+				language = await loadLanguage("cpp")
-// 				query = language.query(cppQuery)
+				query = language.query(cppQuery)
-// 				break
+				break
-// 			case "c":
+			case "c":
-// 			case "h":
+			case "h":
-// 				language = await loadLanguage("c")
+				language = await loadLanguage("c")
-// 				query = language.query(cQuery)
+				query = language.query(cQuery)
-// 				break
+				break
-// 			case "cs":
+			case "cs":
-// 				language = await loadLanguage("c_sharp")
+				language = await loadLanguage("c_sharp")
-// 				query = language.query(csharpQuery)
+				query = language.query(csharpQuery)
-// 				break
+				break
-// 			case "rb":
+			case "rb":
-// 				language = await loadLanguage("ruby")
+				language = await loadLanguage("ruby")
-// 				query = language.query(rubyQuery)
+				query = language.query(rubyQuery)
-// 				break
+				break
-// 			case "java":
+			case "java":
-// 				language = await loadLanguage("java")
+				language = await loadLanguage("java")
-// 				query = language.query(javaQuery)
+				query = language.query(javaQuery)
-// 				break
+				break
-// 			case "php":
+			case "php":
-// 				language = await loadLanguage("php")
+				language = await loadLanguage("php")
-// 				query = language.query(phpQuery)
+				query = language.query(phpQuery)
-// 				break
+				break
-// 			case "swift":
+			case "swift":
-// 				language = await loadLanguage("swift")
+				language = await loadLanguage("swift")
-// 				query = language.query(swiftQuery)
+				query = language.query(swiftQuery)
-// 				break
+				break
-// 			default:
+			default:
-// 				throw new Error(`Unsupported language: ${ext}`)
+				throw new Error(`Unsupported language: ${ext}`)
-// 		}
+		}
-// 		const parser = new Parser()
+		const parser = new Parser()
-// 		parser.setLanguage(language)
+		parser.setLanguage(language)
-// 		parsers[ext] = { parser, query }
+		parsers[ext] = { parser, query }
-// 	}
+	}
-// 	return parsers
+	return parsers
-// }
+}