Refactor

src/ClaudeDev.ts

@@ -27,7 +27,7 @@ import { truncateHalfConversation } from "./utils/context-management"
 import { extractTextFromFile } from "./utils/extract-text"
 import { regexSearchFiles } from "./utils/ripgrep"
 import { parseMentions } from "./utils/context-mentions"
-import { UrlScraper } from "./utils/UrlScraper"
+import { UrlContentFetcher } from "./utils/UrlContentFetcher"
 
 const SYSTEM_PROMPT =
 	async () => `You are Claude Dev, a highly skilled software developer with extensive knowledge in many programming languages, frameworks, design patterns, and best practices.
@@ -251,7 +251,7 @@ export class ClaudeDev {
 	readonly taskId: string
 	private api: ApiHandler
 	private terminalManager: TerminalManager
-	private urlScraper: UrlScraper
+	private urlContentFetcher: UrlContentFetcher
 	private didEditFile: boolean = false
 	private customInstructions?: string
 	private alwaysAllowReadOnly: boolean
@@ -277,7 +277,7 @@ export class ClaudeDev {
 		this.providerRef = new WeakRef(provider)
 		this.api = buildApiHandler(apiConfiguration)
 		this.terminalManager = new TerminalManager()
-		this.urlScraper = new UrlScraper(provider.context)
+		this.urlContentFetcher = new UrlContentFetcher(provider.context)
 		this.customInstructions = customInstructions
 		this.alwaysAllowReadOnly = alwaysAllowReadOnly ?? false
 
@@ -678,7 +678,7 @@ export class ClaudeDev {
 	abortTask() {
 		this.abort = true // will stop any autonomously running promises
 		this.terminalManager.disposeAll()
-		this.urlScraper.closeBrowser()
+		this.urlContentFetcher.closeBrowser()
 	}
 
 	async executeTool(toolName: ToolName, toolInput: any): Promise<[boolean, ToolResponse]> {
@@ -1647,14 +1647,14 @@ ${this.customInstructions.trim()}
 					if (block.type === "text") {
 						return {
 							...block,
-							text: await parseMentions(block.text, cwd, this.urlScraper),
+							text: await parseMentions(block.text, cwd, this.urlContentFetcher),
 						}
 					} else if (block.type === "tool_result") {
 						const isUserMessage = (text: string) => text.includes("<feedback>") || text.includes("<answer>")
 						if (typeof block.content === "string" && isUserMessage(block.content)) {
 							return {
 								...block,
-								content: await parseMentions(block.content, cwd, this.urlScraper),
+								content: await parseMentions(block.content, cwd, this.urlContentFetcher),
 							}
 						} else if (Array.isArray(block.content)) {
 							const parsedContent = await Promise.all(
@@ -1662,7 +1662,7 @@ ${this.customInstructions.trim()}
 									if (contentBlock.type === "text" && isUserMessage(contentBlock.text)) {
 										return {
 											...contentBlock,
-											text: await parseMentions(contentBlock.text, cwd, this.urlScraper),
+											text: await parseMentions(contentBlock.text, cwd, this.urlContentFetcher),
 										}
 									}
 									return contentBlock

src/extension.ts

@@ -3,7 +3,6 @@
 import * as vscode from "vscode"
 import { ClaudeDevProvider } from "./providers/ClaudeDevProvider"
 import delay from "delay"
-import { UrlScraper } from "./utils/UrlScraper"
 
 /*
 Built using https://github.com/microsoft/vscode-webview-ui-toolkit

src/providers/ClaudeDevProvider.ts

@@ -13,7 +13,6 @@ import { getTheme } from "../utils/getTheme"
 import { openFile, openImage } from "../utils/open-file"
 import WorkspaceTracker from "../integrations/WorkspaceTracker"
 import { openMention } from "../utils/context-mentions"
-import { UrlScraper } from "../utils/UrlScraper"
 
 /*
 https://github.com/microsoft/vscode-webview-ui-toolkit-samples/blob/main/default/weather-webview/src/providers/WeatherViewProvider.ts

src/utils/UrlContentFetcher.ts

@@ -12,7 +12,7 @@ interface PCRStats {
 	executablePath: string
 }
 
-export class UrlScraper {
+export class UrlContentFetcher {
 	private context: vscode.ExtensionContext
 	private browser?: Browser
 	private page?: Page
@@ -74,8 +74,7 @@ export class UrlScraper {
 		/*
 		- networkidle2 is equivalent to playwright's networkidle where it waits until there are no more than 2 network connections for at least 500 ms.
 		- domcontentloaded is when the basic DOM is loaded
-		this should be sufficient for most doc sites, but we can use the more elaborate waitTillHTMLRendered if we find users are scraping more dynamic complex sites
+		this should be sufficient for most doc sites
-		https://stackoverflow.com/questions/52497252/puppeteer-wait-until-page-is-completely-loaded/61304202#61304202
 		*/
 		await this.page.goto(url, { timeout: 10_000, waitUntil: ["domcontentloaded", "networkidle2"] })
 		const content = await this.page.content()

src/utils/context-mentions.ts

@@ -1,7 +1,7 @@
 import * as vscode from "vscode"
 import * as path from "path"
 import { openFile } from "./open-file"
-import { UrlScraper } from "./UrlScraper"
+import { UrlContentFetcher } from "./UrlContentFetcher"
 import { mentionRegexGlobal } from "../shared/context-mentions"
 import fs from "fs/promises"
 import { extractTextFromFile } from "./extract-text"
@@ -32,7 +32,7 @@ export function openMention(mention?: string): void {
 	}
 }
 
-export async function parseMentions(text: string, cwd: string, urlScraper: UrlScraper): Promise<string> {
+export async function parseMentions(text: string, cwd: string, urlContentFetcher: UrlContentFetcher): Promise<string> {
 	const mentions: Set<string> = new Set()
 	let parsedText = text.replace(mentionRegexGlobal, (match, mention) => {
 		mentions.add(mention)
@@ -53,7 +53,7 @@ export async function parseMentions(text: string, cwd: string, urlScraper: UrlSc
 	let launchBrowserError: Error | undefined
 	if (urlMention) {
 		try {
-			await urlScraper.launchBrowser()
+			await urlContentFetcher.launchBrowser()
 		} catch (error) {
 			launchBrowserError = error
 			vscode.window.showErrorMessage(`Error fetching content for ${urlMention}: ${error.message}`)
@@ -67,7 +67,7 @@ export async function parseMentions(text: string, cwd: string, urlScraper: UrlSc
 				result = `Error fetching content: ${launchBrowserError.message}`
 			} else {
 				try {
-					const markdown = await urlScraper.urlToMarkdown(mention)
+					const markdown = await urlContentFetcher.urlToMarkdown(mention)
 					result = markdown
 				} catch (error) {
 					vscode.window.showErrorMessage(`Error fetching content for ${mention}: ${error.message}`)
@@ -103,7 +103,7 @@ export async function parseMentions(text: string, cwd: string, urlScraper: UrlSc
 
 	if (urlMention) {
 		try {
-			await urlScraper.closeBrowser()
+			await urlContentFetcher.closeBrowser()
 		} catch (error) {
 			console.error(`Error closing browser: ${error.message}`)
 		}