From b8d55931b6cc79970c6ccc6e2b21117eeacde660 Mon Sep 17 00:00:00 2001 From: "inference-gateway-maintainer[bot]" <246577062+inference-gateway-maintainer[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:57:43 +0000 Subject: [PATCH 1/4] fix: disable take_screenshot tool and remove screenshot references from system prompt for lightpanda engine --- main.go | 52 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index 5357351..36e8e37 100644 --- a/main.go +++ b/main.go @@ -254,10 +254,17 @@ func runStart(ctx context.Context) error { toolBox.AddTool(extractDataTool) l.Info("registered tool: extract_data (Extract data from the page using selectors and return structured information)") - // Register take_screenshot tool - takeScreenshotTool := tools.NewTakeScreenshotTool(l, playwrightSvc) - toolBox.AddTool(takeScreenshotTool) - l.Info("registered tool: take_screenshot (Capture a screenshot of the current page or specific element)") + // Determine if the engine supports screenshots (lightpanda has no graphical rendering engine) + isLightpanda := strings.EqualFold(cfg.Browser.Engine, string(playwright.Lightpanda)) + + // Register take_screenshot tool (only for engines that support it) + if !isLightpanda { + takeScreenshotTool := tools.NewTakeScreenshotTool(l, playwrightSvc) + toolBox.AddTool(takeScreenshotTool) + l.Info("registered tool: take_screenshot (Capture a screenshot of the current page or specific element)") + } else { + l.Info("skipped take_screenshot tool: not supported by lightpanda engine") + } // Register execute_script tool executeScriptTool := tools.NewExecuteScriptTool(l, playwrightSvc) @@ -279,14 +286,23 @@ func runStart(ctx context.Context) error { return fmt.Errorf("failed to create LLM client: %w", err) } - systemPrompt := `You are an expert Playwright browser automation assistant with the ability to create downloadable artifacts. Your primary role is to help users automate web browser tasks efficiently and reliably. + // Build the system prompt, omitting screenshot-related content when the engine does not support it + var systemPrompt string + { + prompt := `You are an expert Playwright browser automation assistant with the ability to create downloadable artifacts. Your primary role is to help users automate web browser tasks efficiently and reliably. Your core capabilities include: 1. **Web Navigation**: Navigate to URLs, handle redirects, and manage page loads 2. **Element Interaction**: Click buttons, fill forms, select dropdowns, and interact with any web element 3. **Data Extraction**: Scrape and extract structured data from web pages -4. **Form Automation**: Fill and submit complex forms with validation -5. **Screenshot Capture**: Take full-page or element-specific screenshots +4. **Form Automation**: Fill and submit complex forms with validation` + + if !isLightpanda { + prompt += ` +5. **Screenshot Capture**: Take full-page or element-specific screenshots` + } + + prompt += ` 6. **JavaScript Execution**: Run custom scripts in the browser context 7. **Authentication Handling**: Manage various authentication methods 8. **Synchronization**: Wait for specific conditions and handle dynamic content @@ -326,21 +342,37 @@ Reach for fetch when: Reach for navigate_to_url (and the Playwright tools) when: - The page is a Single-Page App that hydrates client-side. - Content is behind authentication, cookies, or CSRF that requires a browser session. -- You need to interact with the DOM (click, fill, screenshot). +- You need to interact with the DOM (click, fill,` + + if !isLightpanda { + prompt += ` screenshot).` + } else { + prompt += `).` + } + + prompt += ` - The page renders meaningful content only after JS execution (most modern article sites, dashboards, admin panels). When in doubt: try fetch first. If the response body looks like an empty shell that gets filled in by JS, fall back to navigate_to_url. -**IMPORTANT - Artifact Creation**: +` + + if !isLightpanda { + prompt += `**IMPORTANT - Artifact Creation**: When users request screenshots, the take_screenshot tool automatically creates downloadable artifacts. The screenshot will be available via a download URL returned in the response. -For data extraction, you can use the create_artifact tool to save extracted data as downloadable files (JSON/CSV/TXT). +` + } + + prompt += `For data extraction, you can use the create_artifact tool to save extracted data as downloadable files (JSON/CSV/TXT). **IMPORTANT - Answering capability questions**: When the user asks about your skills, tools, capabilities, or what you can do (e.g. "what skills do you have?", "list your tools", "what can you do?"), answer directly from this system prompt and the AVAILABLE SKILLS list below. Do NOT call any tools, do NOT navigate to a URL, and do NOT Read SKILL.md files. Only load a SKILL.md (via the Read tool) once the user has given you a concrete task that matches one of those skills. Your automation solutions should be maintainable, efficient, and production-ready. ` + systemPrompt = prompt + } if skillsPrompt != "" { systemPrompt = systemPrompt + "\n\n" + skillsPrompt } From dab68635254650a0cdd5dd1cfd7dfc9c7a5e922a Mon Sep 17 00:00:00 2001 From: "inference-gateway-maintainer[bot]" <246577062+inference-gateway-maintainer[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:05:51 +0000 Subject: [PATCH 2/4] fix(adl): protect main.go in .adl-ignore and update manifest/skills for lightpanda screenshot support --- .adl-ignore | 1 + .agents/skills/deep-research/SKILL.md | 5 +++++ .agents/skills/form-automation/SKILL.md | 5 +++++ .agents/skills/web-scraping/SKILL.md | 5 +++++ .agents/skills/webapp-testing/SKILL.md | 5 +++++ agent.yaml | 12 +++++++++--- 6 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.adl-ignore b/.adl-ignore index 6c8a8b2..bade02b 100644 --- a/.adl-ignore +++ b/.adl-ignore @@ -26,4 +26,5 @@ internal/playwright/playwright.go .agents/skills/form-automation/ .agents/skills/deep-research/ +main.go Dockerfile diff --git a/.agents/skills/deep-research/SKILL.md b/.agents/skills/deep-research/SKILL.md index bc37f22..a90ceea 100644 --- a/.agents/skills/deep-research/SKILL.md +++ b/.agents/skills/deep-research/SKILL.md @@ -36,6 +36,11 @@ skill. ## Workflow: plan-then-execute (6 steps) +> **Note:** The `take_screenshot` tool is only available when using Chromium, +> Firefox, or WebKit engines. When the engine is lightpanda (no graphical +> rendering), skip screenshot steps and rely on `extract_data` and +> `execute_script` for DOM inspection. + The most common failure modes in deep research are (a) jumping straight to the first search result, (b) treating N restatements of one primary source as N independent confirmations, and (c) diff --git a/.agents/skills/form-automation/SKILL.md b/.agents/skills/form-automation/SKILL.md index 164ef4e..500fc4c 100644 --- a/.agents/skills/form-automation/SKILL.md +++ b/.agents/skills/form-automation/SKILL.md @@ -31,6 +31,11 @@ Do **not** use this for testing whether a form *works* (use ## Workflow +> **Note:** The `take_screenshot` tool is only available when using Chromium, +> Firefox, or WebKit engines. When the engine is lightpanda (no graphical +> rendering), skip screenshot steps and rely on `extract_data` and +> `execute_script` for DOM inspection. + 1. **Optional: authenticate** - if the form requires login, run `handle_authentication` first. The session carries cookies across subsequent calls. diff --git a/.agents/skills/web-scraping/SKILL.md b/.agents/skills/web-scraping/SKILL.md index fd1087f..8b5e9fa 100644 --- a/.agents/skills/web-scraping/SKILL.md +++ b/.agents/skills/web-scraping/SKILL.md @@ -28,6 +28,11 @@ Do **not** use this for tasks that mutate page state (use ## Workflow +> **Note:** The `take_screenshot` tool is only available when using Chromium, +> Firefox, or WebKit engines. When the engine is lightpanda (no graphical +> rendering), skip screenshot steps and rely on `extract_data` and +> `execute_script` for DOM inspection. + 0. **Skip the browser when you can** - before opening a Playwright session, check if the data is reachable without one: - Does the site expose a JSON/XML API? Many SPAs render from a diff --git a/.agents/skills/webapp-testing/SKILL.md b/.agents/skills/webapp-testing/SKILL.md index b59a4b9..ac4231c 100644 --- a/.agents/skills/webapp-testing/SKILL.md +++ b/.agents/skills/webapp-testing/SKILL.md @@ -30,6 +30,11 @@ or for filling a single form (use `form-automation`). ## Workflow: reconnaissance-then-action +> **Note:** The `take_screenshot` tool is only available when using Chromium, +> Firefox, or WebKit engines. When the engine is lightpanda (no graphical +> rendering), skip screenshot steps and rely on `extract_data` and +> `execute_script` for DOM inspection. + The most common failure mode in webapp testing is acting on stale assumptions about the DOM. Always inspect the rendered page before choosing selectors. diff --git a/agent.yaml b/agent.yaml index b65379a..4185b18 100644 --- a/agent.yaml +++ b/agent.yaml @@ -370,7 +370,8 @@ spec: end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and - take_screenshot." + take_screenshot (only available for chromium/firefox/webkit engines; + lightpanda has no graphical rendering)." tags: - testing - qa @@ -395,8 +396,9 @@ spec: description: "Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, - fill_form, click_element, wait_for_condition, and take_screenshot to - capture the post-submit confirmation." + fill_form, click_element, wait_for_condition, and take_screenshot (only + available for chromium/firefox/webkit engines; lightpanda has no + graphical rendering) to capture the post-submit confirmation." tags: - forms - automation @@ -419,6 +421,10 @@ spec: agent: provider: "" model: "" + # NOTE: The system prompt is built dynamically at runtime (see main.go). + # Screenshot-related content (capability #5, "screenshot" in the DOM + # interaction hint, and the "IMPORTANT - Artifact Creation" section) is + # omitted when the engine is lightpanda, which has no graphical rendering. systemPrompt: | You are an expert Playwright browser automation assistant with the ability to create downloadable artifacts. Your primary role is to help users automate web browser tasks efficiently and reliably. From 51da128065d57aa1cd8a9105fad7a3c43760030d Mon Sep 17 00:00:00 2001 From: Eden Reich Date: Fri, 31 Jul 2026 01:17:18 +0200 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Eden Reich --- agent.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/agent.yaml b/agent.yaml index 4185b18..dc10e08 100644 --- a/agent.yaml +++ b/agent.yaml @@ -421,10 +421,6 @@ spec: agent: provider: "" model: "" - # NOTE: The system prompt is built dynamically at runtime (see main.go). - # Screenshot-related content (capability #5, "screenshot" in the DOM - # interaction hint, and the "IMPORTANT - Artifact Creation" section) is - # omitted when the engine is lightpanda, which has no graphical rendering. systemPrompt: | You are an expert Playwright browser automation assistant with the ability to create downloadable artifacts. Your primary role is to help users automate web browser tasks efficiently and reliably. From d2f9311c17c003eba75462d70007f9d0b24db870 Mon Sep 17 00:00:00 2001 From: "inference-gateway-maintainer[bot]" <246577062+inference-gateway-maintainer[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:22:17 +0000 Subject: [PATCH 4/4] fix(agent): simplify lightpanda handling - add system prompt note, revert skill files, let ADL generate main.go --- .adl-ignore | 1 - .agents/skills/deep-research/SKILL.md | 5 --- .agents/skills/form-automation/SKILL.md | 5 --- .agents/skills/web-scraping/SKILL.md | 5 --- .agents/skills/webapp-testing/SKILL.md | 5 --- .well-known/agent-card.json | 4 +- AGENTS.md | 10 +++-- CLAUDE.md | 4 +- README.md | 4 +- agent.yaml | 2 + main.go | 54 ++++++------------------- 11 files changed, 26 insertions(+), 73 deletions(-) diff --git a/.adl-ignore b/.adl-ignore index bade02b..6c8a8b2 100644 --- a/.adl-ignore +++ b/.adl-ignore @@ -26,5 +26,4 @@ internal/playwright/playwright.go .agents/skills/form-automation/ .agents/skills/deep-research/ -main.go Dockerfile diff --git a/.agents/skills/deep-research/SKILL.md b/.agents/skills/deep-research/SKILL.md index a90ceea..bc37f22 100644 --- a/.agents/skills/deep-research/SKILL.md +++ b/.agents/skills/deep-research/SKILL.md @@ -36,11 +36,6 @@ skill. ## Workflow: plan-then-execute (6 steps) -> **Note:** The `take_screenshot` tool is only available when using Chromium, -> Firefox, or WebKit engines. When the engine is lightpanda (no graphical -> rendering), skip screenshot steps and rely on `extract_data` and -> `execute_script` for DOM inspection. - The most common failure modes in deep research are (a) jumping straight to the first search result, (b) treating N restatements of one primary source as N independent confirmations, and (c) diff --git a/.agents/skills/form-automation/SKILL.md b/.agents/skills/form-automation/SKILL.md index 500fc4c..164ef4e 100644 --- a/.agents/skills/form-automation/SKILL.md +++ b/.agents/skills/form-automation/SKILL.md @@ -31,11 +31,6 @@ Do **not** use this for testing whether a form *works* (use ## Workflow -> **Note:** The `take_screenshot` tool is only available when using Chromium, -> Firefox, or WebKit engines. When the engine is lightpanda (no graphical -> rendering), skip screenshot steps and rely on `extract_data` and -> `execute_script` for DOM inspection. - 1. **Optional: authenticate** - if the form requires login, run `handle_authentication` first. The session carries cookies across subsequent calls. diff --git a/.agents/skills/web-scraping/SKILL.md b/.agents/skills/web-scraping/SKILL.md index 8b5e9fa..fd1087f 100644 --- a/.agents/skills/web-scraping/SKILL.md +++ b/.agents/skills/web-scraping/SKILL.md @@ -28,11 +28,6 @@ Do **not** use this for tasks that mutate page state (use ## Workflow -> **Note:** The `take_screenshot` tool is only available when using Chromium, -> Firefox, or WebKit engines. When the engine is lightpanda (no graphical -> rendering), skip screenshot steps and rely on `extract_data` and -> `execute_script` for DOM inspection. - 0. **Skip the browser when you can** - before opening a Playwright session, check if the data is reachable without one: - Does the site expose a JSON/XML API? Many SPAs render from a diff --git a/.agents/skills/webapp-testing/SKILL.md b/.agents/skills/webapp-testing/SKILL.md index ac4231c..b59a4b9 100644 --- a/.agents/skills/webapp-testing/SKILL.md +++ b/.agents/skills/webapp-testing/SKILL.md @@ -30,11 +30,6 @@ or for filling a single form (use `form-automation`). ## Workflow: reconnaissance-then-action -> **Note:** The `take_screenshot` tool is only available when using Chromium, -> Firefox, or WebKit engines. When the engine is lightpanda (no graphical -> rendering), skip screenshot steps and rely on `extract_data` and -> `execute_script` for DOM inspection. - The most common failure mode in webapp testing is acting on stale assumptions about the DOM. Always inspect the rendered page before choosing selectors. diff --git a/.well-known/agent-card.json b/.well-known/agent-card.json index cc1d56a..05d6b25 100644 --- a/.well-known/agent-card.json +++ b/.well-known/agent-card.json @@ -16,7 +16,7 @@ { "id": "webapp-testing", "name": "webapp-testing", - "description": "Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot.", + "description": "Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering).", "tags": ["testing","qa","e2e","playwright"] }, { @@ -28,7 +28,7 @@ { "id": "form-automation", "name": "form-automation", - "description": "Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot to capture the post-submit confirmation.", + "description": "Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering) to capture the post-submit confirmation.", "tags": ["forms","automation","workflow"] }, { diff --git a/AGENTS.md b/AGENTS.md index 7729dca..adefd92 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,8 @@ For data extraction, you can use the create_artifact tool to save extracted data **IMPORTANT - Answering capability questions**: When the user asks about your skills, tools, capabilities, or what you can do (e.g. "what skills do you have?", "list your tools", "what can you do?"), answer directly from this system prompt and the AVAILABLE SKILLS list below. Do NOT call any tools, do NOT navigate to a URL, and do NOT Read SKILL.md files. Only load a SKILL.md (via the Read tool) once the user has given you a concrete task that matches one of those skills. +**Note**: When the browser engine is configured as lightpanda (no graphical rendering), the take_screenshot tool is not available. Use extract_data and execute_script for DOM inspection instead. + Your automation solutions should be maintainable, efficient, and production-ready. @@ -155,7 +157,7 @@ This agent exposes 12 function-call tools: This agent ships 4 markdown skills that are loaded into the system prompt at startup: ### webapp-testing -- **Description**: Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot. +- **Description**: Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering). - **Tags**: testing, qa, e2e, playwright - **Source**: scaffolded locally (`.agents/skills/webapp-testing/SKILL.md`) @@ -165,7 +167,7 @@ This agent ships 4 markdown skills that are loaded into the system prompt at sta - **Source**: scaffolded locally (`.agents/skills/web-scraping/SKILL.md`) ### form-automation -- **Description**: Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot to capture the post-submit confirmation. +- **Description**: Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering) to capture the post-submit confirmation. - **Tags**: forms, automation, workflow - **Source**: scaffolded locally (`.agents/skills/form-automation/SKILL.md`) @@ -262,11 +264,11 @@ docker run -p 8080:8080 browser-agent │ └── handle_authentication.go # Handle various authentication scenarios including basic auth, OAuth, and custom login forms │ └── wait_for_condition.go # Wait for specific conditions before proceeding with automation ├── .agents/skills/ # Skill directories (SKILL.md + optional assets) -│ └── webapp-testing/ # Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot. +│ └── webapp-testing/ # Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering). │ └── SKILL.md # Playbook prepended to the system prompt │ └── web-scraping/ # Use this when the user asks to extract structured data from one or more pages. Drives extract_data across paginated URLs, normalizes results, and writes a JSON/CSV artifact via the write tool. │ └── SKILL.md # Playbook prepended to the system prompt -│ └── form-automation/ # Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot to capture the post-submit confirmation. +│ └── form-automation/ # Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering) to capture the post-submit confirmation. │ └── SKILL.md # Playbook prepended to the system prompt │ └── deep-research/ # Use this when the user asks an open-ended question that needs synthesis from multiple web sources. Plans sub-questions, drives a search engine, visits and cross-references sources via navigate_to_url + extract_data, and writes a cited markdown report with write. │ └── SKILL.md # Playbook prepended to the system prompt diff --git a/CLAUDE.md b/CLAUDE.md index ed0d1fe..27fe496 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,9 +96,9 @@ To modify tools: ### Skills (markdown system-prompt playbooks) The following skills are currently shipped with the agent: -- **webapp-testing** (bare scaffold): Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot. +- **webapp-testing** (bare scaffold): Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering). - **web-scraping** (bare scaffold): Use this when the user asks to extract structured data from one or more pages. Drives extract_data across paginated URLs, normalizes results, and writes a JSON/CSV artifact via the write tool. -- **form-automation** (bare scaffold): Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot to capture the post-submit confirmation. +- **form-automation** (bare scaffold): Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering) to capture the post-submit confirmation. - **deep-research** (bare scaffold): Use this when the user asks an open-ended question that needs synthesis from multiple web sources. Plans sub-questions, drives a search engine, visits and cross-references sources via navigate_to_url + extract_data, and writes a cited markdown report with write. Each skill lives in its own directory at `.agents/skills//SKILL.md` diff --git a/README.md b/README.md index f1b5241..65a2607 100644 --- a/README.md +++ b/README.md @@ -95,9 +95,9 @@ infer agents add browser-agent http://localhost:8080 \ | Skill | Description | Source | |-------|-------------|--------| -| `webapp-testing` | Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot. | bare scaffold (`.agents/skills/webapp-testing/SKILL.md`) | +| `webapp-testing` | Use this when the user asks to verify, validate, or test a webapp end-to-end. Performs reconnaissance-then-action: navigate, screenshot the rendered DOM, identify selectors, then exercise the flow using navigate_to_url, click_element, fill_form, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering). | bare scaffold (`.agents/skills/webapp-testing/SKILL.md`) | | `web-scraping` | Use this when the user asks to extract structured data from one or more pages. Drives extract_data across paginated URLs, normalizes results, and writes a JSON/CSV artifact via the write tool. | bare scaffold (`.agents/skills/web-scraping/SKILL.md`) | -| `form-automation` | Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot to capture the post-submit confirmation. | bare scaffold (`.agents/skills/form-automation/SKILL.md`) | +| `form-automation` | Use this when the user asks to complete a multi-step form, optionally behind a login. Orchestrates handle_authentication, navigate_to_url, fill_form, click_element, wait_for_condition, and take_screenshot (only available for chromium/firefox/webkit engines; lightpanda has no graphical rendering) to capture the post-submit confirmation. | bare scaffold (`.agents/skills/form-automation/SKILL.md`) | | `deep-research` | Use this when the user asks an open-ended question that needs synthesis from multiple web sources. Plans sub-questions, drives a search engine, visits and cross-references sources via navigate_to_url + extract_data, and writes a cited markdown report with write. | bare scaffold (`.agents/skills/deep-research/SKILL.md`) | ## Documentation diff --git a/agent.yaml b/agent.yaml index dc10e08..b25b320 100644 --- a/agent.yaml +++ b/agent.yaml @@ -482,6 +482,8 @@ spec: **IMPORTANT - Answering capability questions**: When the user asks about your skills, tools, capabilities, or what you can do (e.g. "what skills do you have?", "list your tools", "what can you do?"), answer directly from this system prompt and the AVAILABLE SKILLS list below. Do NOT call any tools, do NOT navigate to a URL, and do NOT Read SKILL.md files. Only load a SKILL.md (via the Read tool) once the user has given you a concrete task that matches one of those skills. + **Note**: When the browser engine is configured as lightpanda (no graphical rendering), the take_screenshot tool is not available. Use extract_data and execute_script for DOM inspection instead. + Your automation solutions should be maintainable, efficient, and production-ready. mcp: enabled: false diff --git a/main.go b/main.go index 36e8e37..d051518 100644 --- a/main.go +++ b/main.go @@ -254,17 +254,10 @@ func runStart(ctx context.Context) error { toolBox.AddTool(extractDataTool) l.Info("registered tool: extract_data (Extract data from the page using selectors and return structured information)") - // Determine if the engine supports screenshots (lightpanda has no graphical rendering engine) - isLightpanda := strings.EqualFold(cfg.Browser.Engine, string(playwright.Lightpanda)) - - // Register take_screenshot tool (only for engines that support it) - if !isLightpanda { - takeScreenshotTool := tools.NewTakeScreenshotTool(l, playwrightSvc) - toolBox.AddTool(takeScreenshotTool) - l.Info("registered tool: take_screenshot (Capture a screenshot of the current page or specific element)") - } else { - l.Info("skipped take_screenshot tool: not supported by lightpanda engine") - } + // Register take_screenshot tool + takeScreenshotTool := tools.NewTakeScreenshotTool(l, playwrightSvc) + toolBox.AddTool(takeScreenshotTool) + l.Info("registered tool: take_screenshot (Capture a screenshot of the current page or specific element)") // Register execute_script tool executeScriptTool := tools.NewExecuteScriptTool(l, playwrightSvc) @@ -286,23 +279,14 @@ func runStart(ctx context.Context) error { return fmt.Errorf("failed to create LLM client: %w", err) } - // Build the system prompt, omitting screenshot-related content when the engine does not support it - var systemPrompt string - { - prompt := `You are an expert Playwright browser automation assistant with the ability to create downloadable artifacts. Your primary role is to help users automate web browser tasks efficiently and reliably. + systemPrompt := `You are an expert Playwright browser automation assistant with the ability to create downloadable artifacts. Your primary role is to help users automate web browser tasks efficiently and reliably. Your core capabilities include: 1. **Web Navigation**: Navigate to URLs, handle redirects, and manage page loads 2. **Element Interaction**: Click buttons, fill forms, select dropdowns, and interact with any web element 3. **Data Extraction**: Scrape and extract structured data from web pages -4. **Form Automation**: Fill and submit complex forms with validation` - - if !isLightpanda { - prompt += ` -5. **Screenshot Capture**: Take full-page or element-specific screenshots` - } - - prompt += ` +4. **Form Automation**: Fill and submit complex forms with validation +5. **Screenshot Capture**: Take full-page or element-specific screenshots 6. **JavaScript Execution**: Run custom scripts in the browser context 7. **Authentication Handling**: Manage various authentication methods 8. **Synchronization**: Wait for specific conditions and handle dynamic content @@ -342,37 +326,23 @@ Reach for fetch when: Reach for navigate_to_url (and the Playwright tools) when: - The page is a Single-Page App that hydrates client-side. - Content is behind authentication, cookies, or CSRF that requires a browser session. -- You need to interact with the DOM (click, fill,` - - if !isLightpanda { - prompt += ` screenshot).` - } else { - prompt += `).` - } - - prompt += ` +- You need to interact with the DOM (click, fill, screenshot). - The page renders meaningful content only after JS execution (most modern article sites, dashboards, admin panels). When in doubt: try fetch first. If the response body looks like an empty shell that gets filled in by JS, fall back to navigate_to_url. -` - - if !isLightpanda { - prompt += `**IMPORTANT - Artifact Creation**: +**IMPORTANT - Artifact Creation**: When users request screenshots, the take_screenshot tool automatically creates downloadable artifacts. The screenshot will be available via a download URL returned in the response. -` - } - - prompt += `For data extraction, you can use the create_artifact tool to save extracted data as downloadable files (JSON/CSV/TXT). +For data extraction, you can use the create_artifact tool to save extracted data as downloadable files (JSON/CSV/TXT). **IMPORTANT - Answering capability questions**: When the user asks about your skills, tools, capabilities, or what you can do (e.g. "what skills do you have?", "list your tools", "what can you do?"), answer directly from this system prompt and the AVAILABLE SKILLS list below. Do NOT call any tools, do NOT navigate to a URL, and do NOT Read SKILL.md files. Only load a SKILL.md (via the Read tool) once the user has given you a concrete task that matches one of those skills. +**Note**: When the browser engine is configured as lightpanda (no graphical rendering), the take_screenshot tool is not available. Use extract_data and execute_script for DOM inspection instead. + Your automation solutions should be maintainable, efficient, and production-ready. ` - systemPrompt = prompt - } if skillsPrompt != "" { systemPrompt = systemPrompt + "\n\n" + skillsPrompt }