More flexibility for LLMs not being great at this

This commit is contained in:
Matt Rubens
2024-12-19 02:12:20 -05:00
parent ef9c468f17
commit 5d930981a4
4 changed files with 105 additions and 162 deletions

View File

@@ -1041,6 +1041,7 @@ export class Cline {
case "write_to_file": { case "write_to_file": {
const relPath: string | undefined = block.params.path const relPath: string | undefined = block.params.path
let newContent: string | undefined = block.params.content let newContent: string | undefined = block.params.content
let predictedLineCount: number | undefined = parseInt(block.params.line_count ?? "0")
if (!relPath || !newContent) { if (!relPath || !newContent) {
// checking for newContent ensure relPath is complete // checking for newContent ensure relPath is complete
// wait so we can determine if it's a new file or editing an existing file // wait so we can determine if it's a new file or editing an existing file
@@ -1109,6 +1110,12 @@ export class Cline {
await this.diffViewProvider.reset() await this.diffViewProvider.reset()
break break
} }
if (!predictedLineCount) {
this.consecutiveMistakeCount++
pushToolResult(await this.sayAndCreateMissingParamError("write_to_file", "line_count"))
await this.diffViewProvider.reset()
break
}
this.consecutiveMistakeCount = 0 this.consecutiveMistakeCount = 0
// if isEditingFile false, that means we have the full contents of the file already. // if isEditingFile false, that means we have the full contents of the file already.
@@ -1125,12 +1132,11 @@ export class Cline {
this.diffViewProvider.scrollToFirstDiff() this.diffViewProvider.scrollToFirstDiff()
// Check for code omissions before proceeding // Check for code omissions before proceeding
const predictedLineCount = parseInt(block.params.line_count ?? "0")
if (detectCodeOmission(this.diffViewProvider.originalContent || "", newContent, predictedLineCount)) { if (detectCodeOmission(this.diffViewProvider.originalContent || "", newContent, predictedLineCount)) {
if (this.diffStrategy) { if (this.diffStrategy) {
await this.diffViewProvider.revertChanges() await this.diffViewProvider.revertChanges()
pushToolResult(formatResponse.toolError( pushToolResult(formatResponse.toolError(
`Content appears to be truncated (file has ${newContent.split("\n").length} lines but was predicted to have ${predictedLineCount} lines). Please provide the complete file content without any omissions if possible, or otherwise use the 'apply_diff' tool to apply the diff to the original file.` `Content appears to be truncated (file has ${newContent.split("\n").length} lines but was predicted to have ${predictedLineCount} lines), and found comments indicating omitted code (e.g., '// rest of code unchanged', '/* previous code */'). Please provide the complete file content without any omissions if possible, or otherwise use the 'apply_diff' tool to apply the diff to the original file.`
)) ))
break break
} else { } else {

View File

@@ -62,15 +62,15 @@ Usage:
Description: Request to write full content to a file at the specified path. If the file exists, it will be overwritten with the provided content. If the file doesn't exist, it will be created. This tool will automatically create any directories needed to write the file. Description: Request to write full content to a file at the specified path. If the file exists, it will be overwritten with the provided content. If the file doesn't exist, it will be created. This tool will automatically create any directories needed to write the file.
Parameters: Parameters:
- path: (required) The path of the file to write to (relative to the current working directory ${cwd.toPosix()}) - path: (required) The path of the file to write to (relative to the current working directory ${cwd.toPosix()})
- line_count: (required) The number of lines in the content. This is used to determine if the user needs to provide more content to complete the file.
- content: (required) The content to write to the file. ALWAYS provide the COMPLETE intended content of the file, without any truncation or omissions. You MUST include ALL parts of the file, even if they haven't been modified. Do NOT include the line numbers in the content though, just the actual content of the file. - content: (required) The content to write to the file. ALWAYS provide the COMPLETE intended content of the file, without any truncation or omissions. You MUST include ALL parts of the file, even if they haven't been modified. Do NOT include the line numbers in the content though, just the actual content of the file.
- line_count: (required) The number of lines in the file. Make sure to compute this based on the actual content of the file, not the number of lines in the content you're providing.
Usage: Usage:
<write_to_file> <write_to_file>
<line_count>total number of lines in the content, including empty lines</line_count>
<path>File path here</path> <path>File path here</path>
<content> <content>
Your file content here Your file content here
</content> </content>
<line_count>total number of lines in the file, including empty lines</line_count>
</write_to_file> </write_to_file>
${diffStrategy ? diffStrategy.getToolDescription(cwd.toPosix()) : ""} ${diffStrategy ? diffStrategy.getToolDescription(cwd.toPosix()) : ""}
@@ -209,7 +209,6 @@ Your final result description here
## Example 2: Requesting to write to a file ## Example 2: Requesting to write to a file
<write_to_file> <write_to_file>
<line_count>14</line_count>
<path>frontend-config.json</path> <path>frontend-config.json</path>
<content> <content>
{ {
@@ -227,6 +226,7 @@ Your final result description here
"version": "1.0.0" "version": "1.0.0"
} }
</content> </content>
<line_count>14</line_count>
</write_to_file> </write_to_file>
## Example 3: Requesting to use an MCP tool ## Example 3: Requesting to use an MCP tool

View File

@@ -8,185 +8,127 @@ describe('detectCodeOmission', () => {
return x + y; return x + y;
}` }`
it('should skip square bracket checks for files under 100 lines', () => { const generateLongContent = (commentLine: string, length: number = 90) => {
const newContent = `[Previous content from line 1-305 remains exactly the same] return `${commentLine}
const z = 3;` ${Array.from({ length }, (_, i) => `const x${i} = ${i};`).join('\n')}
expect(detectCodeOmission(originalContent, newContent)).toBe(false) const y = 2;`
}) }
it('should skip single-line comment checks for files under 100 lines', () => { it('should skip comment checks for files under 100 lines', () => {
const newContent = `// Lines 1-50 remain unchanged const newContent = `// Lines 1-50 remain unchanged
const z = 3;` const z = 3;`
expect(detectCodeOmission(originalContent, newContent)).toBe(false) const predictedLineCount = 50
}) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
it('should skip multi-line comment checks for files under 100 lines', () => {
const newContent = `/* Previous content remains the same */
const z = 3;`
expect(detectCodeOmission(originalContent, newContent)).toBe(false)
})
it('should skip HTML-style comment checks for files under 100 lines', () => {
const newContent = `<!-- Existing content unchanged -->
const z = 3;`
expect(detectCodeOmission(originalContent, newContent)).toBe(false)
})
it('should skip JSX-style comment checks for files under 100 lines', () => {
const newContent = `{/* Rest of the code remains the same */}
const z = 3;`
expect(detectCodeOmission(originalContent, newContent)).toBe(false)
})
it('should skip Python-style comment checks for files under 100 lines', () => {
const newContent = `# Previous content remains unchanged
const z = 3;`
expect(detectCodeOmission(originalContent, newContent)).toBe(false)
}) })
it('should not detect regular comments without omission keywords', () => { it('should not detect regular comments without omission keywords', () => {
const newContent = `// Adding new functionality const newContent = generateLongContent('// Adding new functionality')
const z = 3;` const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
it('should not detect when comment is part of original content', () => { it('should not detect when comment is part of original content', () => {
const originalWithComment = `// Content remains unchanged const originalWithComment = `// Content remains unchanged
${originalContent}` ${originalContent}`
const newContent = `// Content remains unchanged const newContent = generateLongContent('// Content remains unchanged')
const z = 3;` const predictedLineCount = 150
expect(detectCodeOmission(originalWithComment, newContent)).toBe(false) expect(detectCodeOmission(originalWithComment, newContent, predictedLineCount)).toBe(false)
}) })
it('should not detect code that happens to contain omission keywords', () => { it('should not detect code that happens to contain omission keywords', () => {
const newContent = `const remains = 'some value'; const newContent = generateLongContent(`const remains = 'some value';
const unchanged = true;` const unchanged = true;`)
expect(detectCodeOmission(originalContent, newContent)).toBe(false) const predictedLineCount = 150
})
describe('with predicted line count', () => {
describe('length-based detection', () => {
it('should skip length checks for files under 100 lines', () => {
const newContent = `const x = 1;`
const predictedLineCount = 50 // Less than 100 lines
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
it('should detect truncation for files with exactly 100 lines', () => { it('should detect suspicious single-line comment when content is more than 20% shorter', () => {
const newContent = `const x = 1;` const newContent = generateLongContent('// Previous content remains here\nconst x = 1;')
const predictedLineCount = 100 // Exactly 100 lines const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true)
}) })
it('should detect truncation for files with more than 100 lines', () => { it('should not flag suspicious single-line comment when content is less than 20% shorter', () => {
const newContent = `const x = 1;` const newContent = generateLongContent('// Previous content remains here', 130)
const predictedLineCount = 150 // More than 100 lines const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true)
})
})
describe('comment-based detection for large files', () => {
const generateLongContent = (commentLine: string) => {
return `${commentLine}
${Array.from({ length: 90 }, (_, i) => `const x${i} = ${i};`).join('\n')}
const y = 2;`
}
it('should detect suspicious single-line comment when content is more than 15% shorter', () => {
const newContent = `// Previous content remains here
const x = 1;`
const predictedLineCount = 100
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true)
})
it('should not flag suspicious single-line comment when content is less than 15% shorter', () => {
const newContent = generateLongContent('// Previous content remains here')
const predictedLineCount = 100
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
it('should detect suspicious Python-style comment when content is more than 15% shorter', () => { it('should detect suspicious Python-style comment when content is more than 20% shorter', () => {
const newContent = `# Previous content remains here const newContent = generateLongContent('# Previous content remains here\nconst x = 1;')
const x = 1;` const predictedLineCount = 150
const predictedLineCount = 100
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true)
}) })
it('should not flag suspicious Python-style comment when content is less than 15% shorter', () => { it('should not flag suspicious Python-style comment when content is less than 20% shorter', () => {
const newContent = generateLongContent('# Previous content remains here') const newContent = generateLongContent('# Previous content remains here', 130)
const predictedLineCount = 100 const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
it('should detect suspicious multi-line comment when content is more than 15% shorter', () => { it('should detect suspicious multi-line comment when content is more than 20% shorter', () => {
const newContent = `/* Previous content remains the same */ const newContent = generateLongContent('/* Previous content remains the same */\nconst x = 1;')
const x = 1;` const predictedLineCount = 150
const predictedLineCount = 100
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true)
}) })
it('should not flag suspicious multi-line comment when content is less than 15% shorter', () => { it('should not flag suspicious multi-line comment when content is less than 20% shorter', () => {
const newContent = generateLongContent('/* Previous content remains the same */') const newContent = generateLongContent('/* Previous content remains the same */', 130)
const predictedLineCount = 100 const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
it('should detect suspicious JSX comment when content is more than 15% shorter', () => { it('should detect suspicious JSX comment when content is more than 20% shorter', () => {
const newContent = `{/* Rest of the code remains the same */} const newContent = generateLongContent('{/* Rest of the code remains the same */}\nconst x = 1;')
const x = 1;` const predictedLineCount = 150
const predictedLineCount = 100
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true)
}) })
it('should not flag suspicious JSX comment when content is less than 15% shorter', () => { it('should not flag suspicious JSX comment when content is less than 20% shorter', () => {
const newContent = generateLongContent('{/* Rest of the code remains the same */}') const newContent = generateLongContent('{/* Rest of the code remains the same */}', 130)
const predictedLineCount = 100 const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
it('should detect suspicious HTML comment when content is more than 15% shorter', () => { it('should detect suspicious HTML comment when content is more than 20% shorter', () => {
const newContent = `<!-- Existing content unchanged --> const newContent = generateLongContent('<!-- Existing content unchanged -->\nconst x = 1;')
const x = 1;` const predictedLineCount = 150
const predictedLineCount = 100
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true)
}) })
it('should not flag suspicious HTML comment when content is less than 15% shorter', () => { it('should not flag suspicious HTML comment when content is less than 20% shorter', () => {
const newContent = generateLongContent('<!-- Existing content unchanged -->') const newContent = generateLongContent('<!-- Existing content unchanged -->', 130)
const predictedLineCount = 100 const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
it('should detect suspicious square bracket notation when content is more than 15% shorter', () => { it('should detect suspicious square bracket notation when content is more than 20% shorter', () => {
const newContent = `[Previous content from line 1-305 remains exactly the same] const newContent = generateLongContent('[Previous content from line 1-305 remains exactly the same]\nconst x = 1;')
const x = 1;` const predictedLineCount = 150
const predictedLineCount = 100
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(true)
}) })
it('should not flag suspicious square bracket notation when content is less than 15% shorter', () => { it('should not flag suspicious square bracket notation when content is less than 20% shorter', () => {
const newContent = generateLongContent('[Previous content from line 1-305 remains exactly the same]') const newContent = generateLongContent('[Previous content from line 1-305 remains exactly the same]', 130)
const predictedLineCount = 100 const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
})
it('should not flag content very close to predicted length', () => { it('should not flag content very close to predicted length', () => {
const newContent = `const x = 1; const newContent = generateLongContent(`const x = 1;
const y = 2; const y = 2;
// This is a legitimate comment that remains here` // This is a legitimate comment that remains here`, 130)
const predictedLineCount = newContent.split('\n').length // Exact line count match const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
it('should not flag when content is longer than predicted', () => { it('should not flag when content is longer than predicted', () => {
const newContent = `const x = 1; const newContent = generateLongContent(`const x = 1;
const y = 2; const y = 2;
// Previous content remains here but we added more // Previous content remains here but we added more
const z = 3; const z = 3;
const w = 4;` const w = 4;`, 160)
const predictedLineCount = 3 // Content has 4 lines (longer than predicted) const predictedLineCount = 150
expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false) expect(detectCodeOmission(originalContent, newContent, predictedLineCount)).toBe(false)
}) })
}) })
})

View File

@@ -2,13 +2,13 @@
* Detects potential AI-generated code omissions in the given file content. * Detects potential AI-generated code omissions in the given file content.
* @param originalFileContent The original content of the file. * @param originalFileContent The original content of the file.
* @param newFileContent The new content of the file to check. * @param newFileContent The new content of the file to check.
* @param predictedLineCount Optional predicted number of lines in the new content. * @param predictedLineCount The predicted number of lines in the new content.
* @returns True if a potential omission is detected, false otherwise. * @returns True if a potential omission is detected, false otherwise.
*/ */
export function detectCodeOmission( export function detectCodeOmission(
originalFileContent: string, originalFileContent: string,
newFileContent: string, newFileContent: string,
predictedLineCount?: number predictedLineCount: number
): boolean { ): boolean {
// Skip all checks if predictedLineCount is less than 100 // Skip all checks if predictedLineCount is less than 100
if (!predictedLineCount || predictedLineCount < 100) { if (!predictedLineCount || predictedLineCount < 100) {
@@ -18,11 +18,6 @@ export function detectCodeOmission(
const actualLineCount = newFileContent.split("\n").length const actualLineCount = newFileContent.split("\n").length
const lengthRatio = actualLineCount / predictedLineCount const lengthRatio = actualLineCount / predictedLineCount
// If content is more than 25% shorter than predicted, this is suspicious
if (lengthRatio <= 0.75) {
return true
}
const originalLines = originalFileContent.split("\n") const originalLines = originalFileContent.split("\n")
const newLines = newFileContent.split("\n") const newLines = newFileContent.split("\n")
const omissionKeywords = ["remain", "remains", "unchanged", "rest", "previous", "existing", "content", "same", "..."] const omissionKeywords = ["remain", "remains", "unchanged", "rest", "previous", "existing", "content", "same", "..."]
@@ -43,8 +38,8 @@ export function detectCodeOmission(
const words = line.toLowerCase().split(/\s+/) const words = line.toLowerCase().split(/\s+/)
if (omissionKeywords.some((keyword) => words.includes(keyword))) { if (omissionKeywords.some((keyword) => words.includes(keyword))) {
if (!originalLines.includes(line)) { if (!originalLines.includes(line)) {
// For files with 100+ lines, only flag if content is more than 15% shorter // For files with 100+ lines, only flag if content is more than 20% shorter
if (lengthRatio <= 0.85) { if (lengthRatio <= 0.80) {
return true return true
} }
} }