enhance web agent; improve task instructions for URL navigation and screenshot capture, and expand browser action capabilities

Author: dcloud347

argus/tools/ask_web_agent.py

@@ -59,12 +59,17 @@ def execute(self, url: str, question: str = "", **_: Any) -> Content:  # type: i
                 )
             )
         self._call_count += 1
-        task = (
-            f"Navigate to {url} and take a screenshot. "
-            f"Describe in detail what you see on the page and answer the question. "
-            f"If the page appears to be blank or not rendering, check console logs and include any error messages in your description."
-        )
+        task = ""
         if question:
-            task += f" Pay special attention to: {question}"
+            task += f"QUESTION TO ANSWER: {question}\n\n"
+        task += f"URL: {url}\n\n"
+        task += (
+            "Steps:\n"
+            "1. Navigate to the URL and take a screenshot.\n"
+            "2. If the page is blank or shows errors, get console logs and report the exact error messages.\n"
+            "3. If the question requires interaction (clicking buttons, typing values, scrolling), perform those actions and take screenshots to capture the result.\n"
+            "4. Use get_text to read specific values if visual inspection is not precise enough.\n"
+            "5. End your response with a one-line verdict: PASS, FAIL, or PARTIAL — and explain why with specific evidence (values seen, errors found, etc.)."
+        )
         text, screenshot = self._web_agent.run_with_screenshot(Content(text=task))
         return Content(text=text, images=[screenshot] if screenshot else [])

argus/tools/browser.py

@@ -18,7 +18,11 @@ class BrowserTool(Tool):
     """
 
     name = "browser"
-    description = "Control a web browser. Supported actions: navigate, screenshot, click, type, scroll, get_text, get_console_logs."
+    description = (
+        "Control a web browser. Supported actions: "
+        "navigate, screenshot, click, dblclick, hover, drag_and_drop, type, press, scroll, "
+        "reload, get_text, get_console_logs."
+    )
     parameters = {
         "type": "object",
         "properties": {
@@ -28,8 +32,13 @@ class BrowserTool(Tool):
                     "navigate",
                     "screenshot",
                     "click",
+                    "dblclick",
+                    "hover",
+                    "drag_and_drop",
                     "type",
+                    "press",
                     "scroll",
+                    "reload",
                     "get_text",
                     "get_console_logs",
                 ],
@@ -38,12 +47,26 @@ class BrowserTool(Tool):
             "url": {"type": "string", "description": "URL to navigate to (navigate action)."},
             "selector": {
                 "type": "string",
-                "description": "CSS selector for the target element (click, type actions).",
+                "description": (
+                    "CSS selector for the target element "
+                    "(click, dblclick, hover, drag_and_drop source, type, press actions)."
+                ),
+            },
+            "target_selector": {
+                "type": "string",
+                "description": "CSS selector for the drop target element (drag_and_drop action).",
             },
             "text": {
                 "type": "string",
                 "description": "Text to enter into the element (type action).",
             },
+            "key": {
+                "type": "string",
+                "description": (
+                    "Key or key combination to press, e.g. 'Enter', 'ArrowDown', 'Control+a' "
+                    "(press action). Uses Playwright key names."
+                ),
+            },
             "delta_x": {
                 "type": "number",
                 "description": "Horizontal scroll delta in pixels (scroll action).",
@@ -93,18 +116,47 @@ async def _execute_async(self, action: str, **kwargs: Any) -> Content:
             await self._page.click(selector)
             return Content(text=f"Clicked '{selector}'")
 
+        if action == "dblclick":
+            selector = kwargs.get("selector", "")
+            await self._page.dblclick(selector)
+            return Content(text=f"Double-clicked '{selector}'")
+
+        if action == "hover":
+            selector = kwargs.get("selector", "")
+            await self._page.hover(selector)
+            return Content(text=f"Hovered over '{selector}'")
+
+        if action == "drag_and_drop":
+            selector = kwargs.get("selector", "")
+            target_selector = kwargs.get("target_selector", "")
+            await self._page.drag_and_drop(selector, target_selector)
+            return Content(text=f"Dragged '{selector}' to '{target_selector}'")
+
         if action == "type":
             selector = kwargs.get("selector", "")
             text = kwargs.get("text", "")
             await self._page.fill(selector, text)
             return Content(text=f"Typed into '{selector}'")
 
+        if action == "press":
+            selector = kwargs.get("selector", "")
+            key = kwargs.get("key", "")
+            if selector:
+                await self._page.press(selector, key)
+            else:
+                await self._page.keyboard.press(key)
+            return Content(text=f"Pressed '{key}'" + (f" on '{selector}'" if selector else ""))
+
         if action == "scroll":
             delta_x = kwargs.get("delta_x", 0)
             delta_y = kwargs.get("delta_y", 0)
             await self._page.mouse.wheel(delta_x, delta_y)
             return Content(text=f"Scrolled ({delta_x}, {delta_y})")
 
+        if action == "reload":
+            await self._page.reload(wait_until="domcontentloaded")
+            return Content(text="Page reloaded.")
+
         if action == "get_text":
             return Content(text=await self._page.inner_text("body"))
 

config.example.yaml

@@ -45,7 +45,8 @@ agent:
       - Do NOT use `applypatch`, `patch`, or any tool not available in a standard bash environment.
       - After each edit, verify the change took effect with `grep -n` or a small `sed -n` read.
       - If an edit fails twice in a row, switch to a different approach immediately.
-    
+      - NEVER use `node - << 'EOF'` or `python3 - << 'EOF'` to edit files — heredoc + scripting language causes multi-level escaping bugs where replacements silently fail. Use `sed -i` for simple substitutions.
+
     ### Edit files with sed:
       ```bash
       # Replace all occurrences
@@ -92,6 +93,17 @@ web_agent:
   headless: true
   max_steps: 20
   log_dir: logs/
+  system_prompt: |
+    You are a web browsing agent with the ability to navigate pages, take screenshots, click elements, type text, and scroll.
+    
+    TASK to do: Navigate to a URL, take a screenshot, and describe what you see. Report the visual appearance of the page clearly and concisely so the caller can decide whether to make changes.
+    
+    General rules:
+    - Always navigate to the URL first, then take a screenshot before drawing conclusions.
+    - If the server is unreachable or the page doesn't appear as expected, check console log and report that immediately.
+    - If the question involves interaction (clicking a button, typing a value, scrolling), perform those actions and take a screenshot after each one.
+    - Base your judgement on visual appearance and behaviour, not on source code.
+    - If the page shows a blank white page or an error message, check console log and report that clearly.
   LLM:
     provider: openai
     model: gpt-5

run_swebench_multimodal.py

@@ -73,7 +73,7 @@ def _build_task(problem_statement: str, port: int, max_steps: int) -> str:
     issue description:
     {problem_statement}
     
-    Please make changes to the codebase and fix the issue!
+    You MUST make changes to the source code and fix the issue!
 """