Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ If you run into any issues, checkout our [troubleshooting guide](./docs/troubles

<!-- BEGIN AUTO GENERATED TOOLS -->

- **Input automation** (10 tools)
- **Input automation** (11 tools)
- [`click`](docs/tool-reference.md#click)
- [`drag`](docs/tool-reference.md#drag)
- [`fill`](docs/tool-reference.md#fill)
Expand All @@ -488,6 +488,7 @@ If you run into any issues, checkout our [troubleshooting guide](./docs/troubles
- [`type_text`](docs/tool-reference.md#type_text)
- [`upload_file`](docs/tool-reference.md#upload_file)
- [`click_at`](docs/tool-reference.md#click_at)
- [`get_element_at`](docs/tool-reference.md#get_element_at)
- **Navigation automation** (6 tools)
- [`close_page`](docs/tool-reference.md#close_page)
- [`list_pages`](docs/tool-reference.md#list_pages)
Expand Down
18 changes: 17 additions & 1 deletion docs/tool-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# Chrome DevTools MCP Tool Reference

- **[Input automation](#input-automation)** (10 tools)
- **[Input automation](#input-automation)** (11 tools)
- [`click`](#click)
- [`drag`](#drag)
- [`fill`](#fill)
Expand All @@ -13,6 +13,7 @@
- [`type_text`](#type_text)
- [`upload_file`](#upload_file)
- [`click_at`](#click_at)
- [`get_element_at`](#get_element_at)
- **[Navigation automation](#navigation-automation)** (6 tools)
- [`close_page`](#close_page)
- [`list_pages`](#list_pages)
Expand Down Expand Up @@ -175,6 +176,21 @@

---

### `get_element_at`

**Description:** Returns the DOM element at viewport-relative CSS-pixel coordinates (x, y). Pairs with [`take_screenshot`](#take_screenshot) + a vision model that emits coordinates. Pierces open shadow roots by default. Limitations: cannot enter closed shadow roots; cannot enter cross-origin/OOPIF iframes (the call returns a 'cross-origin-blocked' result with partial metadata about the iframe); css="matched" requires the experimentalVision flag and uses Chrome DevTools Protocol. For huge elements use mode="schema" (default) or pass filePath to write the full descriptor to disk. (requires flag: --experimentalVision=true)

**Parameters:**

- **x** (number) **(required)**: CSS-pixel X coordinate, viewport-relative.
- **y** (number) **(required)**: CSS-pixel Y coordinate, viewport-relative.
- **css** (enum: "none", "matched", "computed-visual", "computed-full") _(optional)_: CSS data to include. matched = author rules from cascade (uses CDP). computed-visual = ~30 visually relevant computed properties. computed-full = all computed properties (saved to file when large).
- **filePath** (string) _(optional)_: If set, writes the full descriptor (raw outerHTML + full computed CSS) to this path and returns a summary in the response.
- **mode** (enum: "auto", "schema", "raw", "selector-only") _(optional)_: Output detail level. auto/schema = compact MD descriptor. raw = full outerHTML (truncated to 50KB or saved to file). selector-only = just the CSS selector.
- **pierceShadow** (boolean) _(optional)_: Whether to descend into open shadow roots. Default true. Closed shadow roots are never pierced.

---

## Navigation automation

### `close_page`
Expand Down
89 changes: 89 additions & 0 deletions scripts/eval_scenarios/get_element_at_test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/

import assert from 'node:assert';

import type {TestScenario} from '../eval_gemini.ts';

export const scenario: TestScenario = {
serverArgs: ['--experimentalVision=true'],
prompt: `Take a screenshot of <TEST_URL>. There is a single large blue square on the page. Use the get_element_at tool to inspect the DOM element at the center of that blue square (the page is 800x600 and the square spans roughly x=100..300, y=100..300, so a coordinate around 200,200 is appropriate). Then tell me the element's id and class.`,
maxTurns: 4,
htmlRoute: {
path: '/get_element_at_test.html',
htmlContent: `
<!doctype html>
<html>
<head>
<style>
body { margin: 0; background: #ffffff; }
#target {
position: absolute;
left: 100px;
top: 100px;
width: 200px;
height: 200px;
background: #1a73e8;
color: white;
display: flex;
align-items: center;
justify-content: center;
font-size: 24px;
}
</style>
</head>
<body>
<div id="target" class="cta-button" data-testid="primary">CLICK ME</div>
</body>
</html>
`,
},
expectations: calls => {
const visualCalls = calls.filter(
c => c.name === 'take_screenshot' || c.name === 'take_snapshot',
);
assert.ok(
visualCalls.length >= 1,
'Expected at least one take_screenshot or take_snapshot call before inspecting coordinates',
);

const elementAtCalls = calls.filter(c => c.name === 'get_element_at');
assert.ok(
elementAtCalls.length >= 1,
'Expected at least one get_element_at call',
);

let withinTarget = 0;
for (const call of elementAtCalls) {
const x = call.args.x;
const y = call.args.y;
assert.strictEqual(
typeof x,
'number',
'get_element_at must receive a numeric x',
);
assert.strictEqual(
typeof y,
'number',
'get_element_at must receive a numeric y',
);
if (
typeof x === 'number' &&
typeof y === 'number' &&
x >= 100 &&
x <= 300 &&
y >= 100 &&
y <= 300
) {
withinTarget++;
}
}
assert.ok(
withinTarget >= 1,
'Expected at least one get_element_at call with x in [100,300] and y in [100,300] (inside the blue square)',
);
},
};
51 changes: 51 additions & 0 deletions src/bin/chrome-devtools-cli-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,57 @@ export const commands: Commands = {
},
},
},
get_element_at: {
description:
'Returns the DOM element at viewport-relative CSS-pixel coordinates (x, y). Pairs with take_screenshot + a vision model that emits coordinates. Pierces open shadow roots by default. Limitations: cannot enter closed shadow roots; cannot enter cross-origin/OOPIF iframes (the call returns a \'cross-origin-blocked\' result with partial metadata about the iframe); css="matched" requires the experimentalVision flag and uses Chrome DevTools Protocol. For huge elements use mode="schema" (default) or pass filePath to write the full descriptor to disk. (requires flag: --experimentalVision=true)',
category: 'Input automation',
args: {
x: {
name: 'x',
type: 'number',
description: 'CSS-pixel X coordinate, viewport-relative.',
required: true,
},
y: {
name: 'y',
type: 'number',
description: 'CSS-pixel Y coordinate, viewport-relative.',
required: true,
},
mode: {
name: 'mode',
type: 'string',
description:
'Output detail level. auto/schema = compact MD descriptor. raw = full outerHTML (truncated to 50KB or saved to file). selector-only = just the CSS selector.',
required: false,
default: 'auto',
enum: ['auto', 'schema', 'raw', 'selector-only'],
},
css: {
name: 'css',
type: 'string',
description:
'CSS data to include. matched = author rules from cascade (uses CDP). computed-visual = ~30 visually relevant computed properties. computed-full = all computed properties (saved to file when large).',
required: false,
default: 'none',
enum: ['none', 'matched', 'computed-visual', 'computed-full'],
},
pierceShadow: {
name: 'pierceShadow',
type: 'boolean',
description:
'Whether to descend into open shadow roots. Default true. Closed shadow roots are never pierced.',
required: false,
},
filePath: {
name: 'filePath',
type: 'string',
description:
'If set, writes the full descriptor (raw outerHTML + full computed CSS) to this path and returns a summary in the response.',
required: false,
},
},
},
get_memory_snapshot_details: {
description:
'Loads a memory heapsnapshot and returns all available information including statistics, static data, and aggregated node information. Supports pagination for aggregates. (requires flag: --experimentalMemory=true)',
Expand Down
29 changes: 29 additions & 0 deletions src/telemetry/tool_call_metrics.json
Original file line number Diff line number Diff line change
Expand Up @@ -627,5 +627,34 @@
{
"name": "list_3p_developer_tools",
"args": []
},
{
"name": "get_element_at",
"args": [
{
"name": "x",
"argType": "number"
},
{
"name": "y",
"argType": "number"
},
{
"name": "mode",
"argType": "string"
},
{
"name": "css",
"argType": "string"
},
{
"name": "pierce_shadow",
"argType": "boolean"
},
{
"name": "file_path_length",
"argType": "number"
}
]
}
]
Loading