diff --git a/src/ClaudeDev.ts b/src/ClaudeDev.ts index d9e7473..462c0c2 100644 --- a/src/ClaudeDev.ts +++ b/src/ClaudeDev.ts @@ -27,7 +27,7 @@ import { truncateHalfConversation } from "./utils/context-management" import { extractTextFromFile } from "./utils/extract-text" import { regexSearchFiles } from "./utils/ripgrep" import { parseMentions } from "./utils/context-mentions" -import { UrlScraper } from "./utils/UrlScraper" +import { UrlContentFetcher } from "./utils/UrlContentFetcher" const SYSTEM_PROMPT = async () => `You are Claude Dev, a highly skilled software developer with extensive knowledge in many programming languages, frameworks, design patterns, and best practices. @@ -251,7 +251,7 @@ export class ClaudeDev { readonly taskId: string private api: ApiHandler private terminalManager: TerminalManager - private urlScraper: UrlScraper + private urlContentFetcher: UrlContentFetcher private didEditFile: boolean = false private customInstructions?: string private alwaysAllowReadOnly: boolean @@ -277,7 +277,7 @@ export class ClaudeDev { this.providerRef = new WeakRef(provider) this.api = buildApiHandler(apiConfiguration) this.terminalManager = new TerminalManager() - this.urlScraper = new UrlScraper(provider.context) + this.urlContentFetcher = new UrlContentFetcher(provider.context) this.customInstructions = customInstructions this.alwaysAllowReadOnly = alwaysAllowReadOnly ?? false @@ -678,7 +678,7 @@ export class ClaudeDev { abortTask() { this.abort = true // will stop any autonomously running promises this.terminalManager.disposeAll() - this.urlScraper.closeBrowser() + this.urlContentFetcher.closeBrowser() } async executeTool(toolName: ToolName, toolInput: any): Promise<[boolean, ToolResponse]> { @@ -1647,14 +1647,14 @@ ${this.customInstructions.trim()} if (block.type === "text") { return { ...block, - text: await parseMentions(block.text, cwd, this.urlScraper), + text: await parseMentions(block.text, cwd, this.urlContentFetcher), } } else if (block.type === "tool_result") { const isUserMessage = (text: string) => text.includes("") || text.includes("") if (typeof block.content === "string" && isUserMessage(block.content)) { return { ...block, - content: await parseMentions(block.content, cwd, this.urlScraper), + content: await parseMentions(block.content, cwd, this.urlContentFetcher), } } else if (Array.isArray(block.content)) { const parsedContent = await Promise.all( @@ -1662,7 +1662,7 @@ ${this.customInstructions.trim()} if (contentBlock.type === "text" && isUserMessage(contentBlock.text)) { return { ...contentBlock, - text: await parseMentions(contentBlock.text, cwd, this.urlScraper), + text: await parseMentions(contentBlock.text, cwd, this.urlContentFetcher), } } return contentBlock diff --git a/src/extension.ts b/src/extension.ts index 5d89006..a73b861 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -3,7 +3,6 @@ import * as vscode from "vscode" import { ClaudeDevProvider } from "./providers/ClaudeDevProvider" import delay from "delay" -import { UrlScraper } from "./utils/UrlScraper" /* Built using https://github.com/microsoft/vscode-webview-ui-toolkit diff --git a/src/providers/ClaudeDevProvider.ts b/src/providers/ClaudeDevProvider.ts index 891df3f..b710bf0 100644 --- a/src/providers/ClaudeDevProvider.ts +++ b/src/providers/ClaudeDevProvider.ts @@ -13,7 +13,6 @@ import { getTheme } from "../utils/getTheme" import { openFile, openImage } from "../utils/open-file" import WorkspaceTracker from "../integrations/WorkspaceTracker" import { openMention } from "../utils/context-mentions" -import { UrlScraper } from "../utils/UrlScraper" /* https://github.com/microsoft/vscode-webview-ui-toolkit-samples/blob/main/default/weather-webview/src/providers/WeatherViewProvider.ts diff --git a/src/utils/UrlScraper.ts b/src/utils/UrlContentFetcher.ts similarity index 90% rename from src/utils/UrlScraper.ts rename to src/utils/UrlContentFetcher.ts index 50aa8b5..025a303 100644 --- a/src/utils/UrlScraper.ts +++ b/src/utils/UrlContentFetcher.ts @@ -12,7 +12,7 @@ interface PCRStats { executablePath: string } -export class UrlScraper { +export class UrlContentFetcher { private context: vscode.ExtensionContext private browser?: Browser private page?: Page @@ -74,8 +74,7 @@ export class UrlScraper { /* - networkidle2 is equivalent to playwright's networkidle where it waits until there are no more than 2 network connections for at least 500 ms. - domcontentloaded is when the basic DOM is loaded - this should be sufficient for most doc sites, but we can use the more elaborate waitTillHTMLRendered if we find users are scraping more dynamic complex sites - https://stackoverflow.com/questions/52497252/puppeteer-wait-until-page-is-completely-loaded/61304202#61304202 + this should be sufficient for most doc sites */ await this.page.goto(url, { timeout: 10_000, waitUntil: ["domcontentloaded", "networkidle2"] }) const content = await this.page.content() diff --git a/src/utils/context-mentions.ts b/src/utils/context-mentions.ts index bc59a73..833510e 100644 --- a/src/utils/context-mentions.ts +++ b/src/utils/context-mentions.ts @@ -1,7 +1,7 @@ import * as vscode from "vscode" import * as path from "path" import { openFile } from "./open-file" -import { UrlScraper } from "./UrlScraper" +import { UrlContentFetcher } from "./UrlContentFetcher" import { mentionRegexGlobal } from "../shared/context-mentions" import fs from "fs/promises" import { extractTextFromFile } from "./extract-text" @@ -32,7 +32,7 @@ export function openMention(mention?: string): void { } } -export async function parseMentions(text: string, cwd: string, urlScraper: UrlScraper): Promise { +export async function parseMentions(text: string, cwd: string, urlContentFetcher: UrlContentFetcher): Promise { const mentions: Set = new Set() let parsedText = text.replace(mentionRegexGlobal, (match, mention) => { mentions.add(mention) @@ -53,7 +53,7 @@ export async function parseMentions(text: string, cwd: string, urlScraper: UrlSc let launchBrowserError: Error | undefined if (urlMention) { try { - await urlScraper.launchBrowser() + await urlContentFetcher.launchBrowser() } catch (error) { launchBrowserError = error vscode.window.showErrorMessage(`Error fetching content for ${urlMention}: ${error.message}`) @@ -67,7 +67,7 @@ export async function parseMentions(text: string, cwd: string, urlScraper: UrlSc result = `Error fetching content: ${launchBrowserError.message}` } else { try { - const markdown = await urlScraper.urlToMarkdown(mention) + const markdown = await urlContentFetcher.urlToMarkdown(mention) result = markdown } catch (error) { vscode.window.showErrorMessage(`Error fetching content for ${mention}: ${error.message}`) @@ -103,7 +103,7 @@ export async function parseMentions(text: string, cwd: string, urlScraper: UrlSc if (urlMention) { try { - await urlScraper.closeBrowser() + await urlContentFetcher.closeBrowser() } catch (error) { console.error(`Error closing browser: ${error.message}`) }