Incoming

Author: KillianLucas

interpreter/computer_use/loop.py

@@ -1,7 +1,10 @@
 import asyncio
 import os
+import pty
+import sys
 from typing import ClassVar, Literal
 
+import pyte
 from anthropic.types.beta import BetaToolBash20241022Param
 
 from .base import BaseAnthropicTool, CLIResult, ToolError, ToolResult
@@ -21,20 +24,43 @@ class _BashSession:
     def __init__(self):
         self._started = False
         self._timed_out = False
+        # Create a terminal screen and stream
+        self._screen = pyte.Screen(80, 24)  # Standard terminal size
+        self._stream = pyte.Stream(self._screen)
 
     async def start(self):
         if self._started:
             return
 
-        self._process = await asyncio.create_subprocess_shell(
-            self.command,
-            preexec_fn=os.setsid,
-            shell=True,
-            bufsize=0,
-            stdin=asyncio.subprocess.PIPE,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
+        try:
+            # Try to create process with PTY
+            master, slave = pty.openpty()
+            self._process = await asyncio.create_subprocess_shell(
+                self.command,
+                preexec_fn=os.setsid,
+                shell=True,
+                bufsize=0,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=slave,
+                stderr=slave,
+            )
+            # Store master fd for reading
+            self._master_fd = master
+            self._using_pty = True
+            print("using pty")
+        except (ImportError, OSError):
+            print("using pipes")
+            # Fall back to regular pipes if PTY is not available
+            self._process = await asyncio.create_subprocess_shell(
+                self.command,
+                preexec_fn=os.setsid,
+                shell=True,
+                bufsize=0,
+                stdin=asyncio.subprocess.PIPE,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            self._using_pty = False
 
         self._started = True
 
@@ -45,18 +71,11 @@ def stop(self):
         if self._process.returncode is not None:
             return
         self._process.terminate()
+        if hasattr(self, "_master_fd"):
+            os.close(self._master_fd)
 
     async def run(self, command: str):
         """Execute a command in the bash shell."""
-        # Ask for user permission before executing the command
-        print(f"Do you want to execute the following command?\n{command}")
-        user_input = input("Enter 'yes' to proceed, anything else to cancel: ")
-
-        if user_input.lower() != "yes":
-            return ToolResult(
-                system="Command execution cancelled by user",
-                error="User did not provide permission to execute the command.",
-            )
         if not self._started:
             raise ToolError("Session has not started.")
         if self._process.returncode is not None:
@@ -71,29 +90,70 @@ async def run(self, command: str):
 
         # we know these are not None because we created the process with PIPEs
         assert self._process.stdin
-        assert self._process.stdout
-        assert self._process.stderr
 
         # send command to the process
         self._process.stdin.write(
             command.encode() + f"; echo '{self._sentinel}'\n".encode()
         )
         await self._process.stdin.drain()
 
-        # read output from the process, until the sentinel is found
         try:
             async with asyncio.timeout(self._timeout):
-                while True:
-                    await asyncio.sleep(self._output_delay)
-                    # if we read directly from stdout/stderr, it will wait forever for
-                    # EOF. use the StreamReader buffer directly instead.
-                    output = (
-                        self._process.stdout._buffer.decode()
-                    )  # pyright: ignore[reportAttributeAccessIssue]
-                    if self._sentinel in output:
-                        # strip the sentinel and break
-                        output = output[: output.index(self._sentinel)]
-                        break
+                if self._using_pty:
+                    # Reset screen state
+                    self._screen.reset()
+                    output = ""
+                    while True:
+                        try:
+                            raw_chunk = os.read(self._master_fd, 1024)
+                            chunk_str = raw_chunk.decode()
+
+                            # Update output before checking sentinel
+                            output += chunk_str
+
+                            # Check for sentinel
+                            if self._sentinel in chunk_str:
+                                # Clean the output for display
+                                clean_chunk = chunk_str[
+                                    : chunk_str.index(self._sentinel)
+                                ].encode()
+                                if clean_chunk:
+                                    os.write(sys.stdout.fileno(), clean_chunk)
+                                # Clean the stored output
+                                if self._sentinel in output:
+                                    output = output[: output.index(self._sentinel)]
+                                break
+
+                            os.write(sys.stdout.fileno(), raw_chunk)
+                        except OSError:
+                            break
+                        await asyncio.sleep(0.01)
+                    error = ""
+                else:
+                    # Real-time output for pipe-based reading
+                    output = ""
+                    while True:
+                        chunk = await self._process.stdout.read(1024)
+                        if not chunk:
+                            break
+                        chunk_str = chunk.decode()
+                        output += chunk_str
+
+                        # Check for sentinel
+                        if self._sentinel in chunk_str:
+                            # Clean the chunk for display
+                            clean_chunk = chunk_str[
+                                : chunk_str.index(self._sentinel)
+                            ].encode()
+                            if clean_chunk:
+                                os.write(sys.stdout.fileno(), clean_chunk)
+                            # Clean the stored output
+                            if self._sentinel in output:
+                                output = output[: output.index(self._sentinel)]
+                            break
+
+                        os.write(sys.stdout.fileno(), chunk)
+                        await asyncio.sleep(0.01)
         except asyncio.TimeoutError:
             self._timed_out = True
             raise ToolError(
@@ -102,19 +162,24 @@ async def run(self, command: str):
 
         if output.endswith("\n"):
             output = output[:-1]
-
-        error = (
-            self._process.stderr._buffer.decode()
-        )  # pyright: ignore[reportAttributeAccessIssue]
-        if error.endswith("\n"):
+        if not self._using_pty and error.endswith("\n"):
             error = error[:-1]
 
-        # clear the buffers so that the next output can be read correctly
-        self._process.stdout._buffer.clear()  # pyright: ignore[reportAttributeAccessIssue]
-        self._process.stderr._buffer.clear()  # pyright: ignore[reportAttributeAccessIssue]
+        # Clear buffers only when using pipes
+        if not self._using_pty:
+            self._process.stdout._buffer.clear()
+            self._process.stderr._buffer.clear()
 
         return CLIResult(output=output, error=error)
 
+    @staticmethod
+    def _strip_ansi(text: str) -> str:
+        """Remove ANSI escape sequences from text."""
+        import re
+
+        ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+        return ansi_escape.sub("", text)
+
 
 class BashTool(BaseAnthropicTool):
     """

interpreter/computer_use/tools/bash.py

@@ -3,24 +3,21 @@
 import math
 import os
 import platform
-import shlex
-import shutil
 import tempfile
 import time
 from enum import StrEnum
 from pathlib import Path
 from typing import Literal, TypedDict
 from uuid import uuid4
 
-# Add import for PyAutoGUI
 import pyautogui
 from anthropic.types.beta import BetaToolComputerUse20241022Param
+from PIL import Image
+from screeninfo import get_monitors
 
 from .base import BaseAnthropicTool, ToolError, ToolResult
 from .run import run
 
-OUTPUT_DIR = "/tmp/outputs"
-
 TYPING_DELAY_MS = 12
 TYPING_GROUP_SIZE = 50
 
@@ -93,15 +90,15 @@ def smooth_move_to(x, y, duration=1.2):
 
 class ComputerTool(BaseAnthropicTool):
     """
-    A tool that allows the agent to interact with the primary monitor's screen, keyboard, and mouse.
+    A tool that allows the agent to interact with the screen, keyboard, and mouse of the current computer.
     The tool parameters are defined by Anthropic and are not editable.
     """
 
     name: Literal["computer"] = "computer"
     api_type: Literal["computer_20241022"] = "computer_20241022"
     width: int
     height: int
-    display_num: None  # Simplified to always be None since we're only using primary display
+    display_num: int | None
 
     _screenshot_delay = 2.0
     _scaling_enabled = True
@@ -122,8 +119,25 @@ def to_params(self) -> BetaToolComputerUse20241022Param:
 
     def __init__(self):
         super().__init__()
+
         self.width, self.height = pyautogui.size()
+
+        # Get display number and set up display offset
         self.display_num = None
+        self._display_offset_x = 0
+        if (display_num := os.getenv("DISPLAY_NUM")) is not None:
+            self.display_num = int(display_num)
+            # Get all displays using screeninfo
+            try:
+                monitors = get_monitors()
+                # Calculate x offset based on display number
+                # Assuming displays are arranged horizontally
+                self._display_offset_x = sum(
+                    m.width for m in monitors[: self.display_num]
+                )
+            except ImportError:
+                # Fallback if screeninfo not available
+                self._display_offset_x = 0
 
     async def __call__(
         self,
@@ -136,15 +150,23 @@ async def __call__(
         if action in ("mouse_move", "left_click_drag"):
             if coordinate is None:
                 raise ToolError(f"coordinate is required for {action}")
+            if text is not None:
+                raise ToolError(f"text is not accepted for {action}")
+            if not isinstance(coordinate, list) or len(coordinate) != 2:
+                raise ToolError(f"{coordinate} must be a tuple of length 2")
+            if not all(isinstance(i, int) and i >= 0 for i in coordinate):
+                raise ToolError(f"{coordinate} must be a tuple of non-negative ints")
+
             x, y = self.scale_coordinates(
                 ScalingSource.API, coordinate[0], coordinate[1]
             )
 
             if action == "mouse_move":
                 smooth_move_to(x, y)
             elif action == "left_click_drag":
+                pyautogui.mouseDown(button="left")
                 smooth_move_to(x, y)
-                pyautogui.dragTo(x, y, button="left")
+                pyautogui.mouseUp(button="left")
 
         elif action in ("key", "type"):
             if text is None:
@@ -232,7 +254,6 @@ async def screenshot(self):
                 ScalingSource.COMPUTER, self.width, self.height
             )
             # Use PIL directly instead of shell convert command
-            from PIL import Image
 
             with Image.open(path) as img:
                 img = img.resize((x, y), Image.Resampling.LANCZOS)

interpreter/computer_use/tools/computer.py

@@ -50,40 +50,18 @@ async def __call__(
         insert_line: int | None = None,
         **kwargs,
     ):
-        # Ask for user permission before executing the command
-        print(f"Do you want to execute the following command?")
-        print(f"Command: {command}")
-        print(f"Path: {path}")
-        if file_text:
-            print(f"File text: {file_text}")
-        if view_range:
-            print(f"View range: {view_range}")
-        if old_str:
-            print(f"Old string: {old_str}")
-        if new_str:
-            print(f"New string: {new_str}")
-        if insert_line is not None:
-            print(f"Insert line: {insert_line}")
-
-        user_input = input("Enter 'yes' to proceed, anything else to cancel: ")
-
-        if user_input.lower() != "yes":
-            return ToolResult(
-                system="Command execution cancelled by user",
-                error="User did not provide permission to execute the command.",
-            )
         _path = Path(path)
         self.validate_path(command, _path)
         if command == "view":
             return await self.view(_path, view_range)
         elif command == "create":
-            if not file_text:
+            if file_text is None:
                 raise ToolError("Parameter `file_text` is required for command: create")
             self.write_file(_path, file_text)
             self._file_history[_path].append(file_text)
             return ToolResult(output=f"File created successfully at: {_path}")
         elif command == "str_replace":
-            if not old_str:
+            if old_str is None:
                 raise ToolError(
                     "Parameter `old_str` is required for command: str_replace"
                 )
@@ -93,7 +71,7 @@ async def __call__(
                 raise ToolError(
                     "Parameter `insert_line` is required for command: insert"
                 )
-            if not new_str:
+            if new_str is None:
                 raise ToolError("Parameter `new_str` is required for command: insert")
             return self.insert(_path, insert_line, new_str)
         elif command == "undo_edit":
@@ -155,15 +133,15 @@ async def view(self, path: Path, view_range: list[int] | None = None):
             init_line, final_line = view_range
             if init_line < 1 or init_line > n_lines_file:
                 raise ToolError(
-                    f"Invalid `view_range`: {view_range}. It's first element `{init_line}` should be within the range of lines of the file: {[1, n_lines_file]}"
+                    f"Invalid `view_range`: {view_range}. Its first element `{init_line}` should be within the range of lines of the file: {[1, n_lines_file]}"
                 )
             if final_line > n_lines_file:
                 raise ToolError(
-                    f"Invalid `view_range`: {view_range}. It's second element `{final_line}` should be smaller than the number of lines in the file: `{n_lines_file}`"
+                    f"Invalid `view_range`: {view_range}. Its second element `{final_line}` should be smaller than the number of lines in the file: `{n_lines_file}`"
                 )
             if final_line != -1 and final_line < init_line:
                 raise ToolError(
-                    f"Invalid `view_range`: {view_range}. It's second element `{final_line}` should be larger or equal than its first `{init_line}`"
+                    f"Invalid `view_range`: {view_range}. Its second element `{final_line}` should be larger or equal than its first `{init_line}`"
                 )
 
             if final_line == -1: