New interpreter --os powered by Anthropic

Author: KillianLucas

interpreter/computer_use/loop.py

@@ -94,7 +94,7 @@ class APIProvider(StrEnum):
 * If the item you are looking at is a pdf, if after taking a single screenshot of the pdf it seems that you want to read the entire document instead of trying to continue to read the pdf from your screenshots + navigation, determine the URL, use curl to download the pdf, install and use pdftotext to convert it to a text file, and then read that text file directly with your StrReplaceEditTool.
 </IMPORTANT>"""
 
-X_SYSTEM_PROMPT = f"""<SYSTEM_CAPABILITY>
+SYSTEM_PROMPT = f"""<SYSTEM_CAPABILITY>
 * You are an AI assistant with access to a virtual machine running on {platform.machine()} architecture.
 * You have the capability to edit and run code.
 * You can use the bash tool to run shell commands, execute scripts, and manage files.
@@ -104,6 +104,18 @@ class APIProvider(StrEnum):
 * The current date is {datetime.today().strftime('%A, %B %-d, %Y')}.
 </SYSTEM_CAPABILITY>"""
 
+SYSTEM_PROMPT = f"""<SYSTEM_CAPABILITY>
+* You are an AI assistant with access to a virtual machine running on {"Mac OS" if platform.system() == "Darwin" else platform.system()} with internet access.
+* When using your computer function calls, they take a while to run and send back to you. Where possible/feasible, try to chain multiple of these calls all into one function calls request.
+</SYSTEM_CAPABILITY>"""
+
+# Update the SYSTEM_PROMPT for Mac OS
+if platform.system() == "Darwin":
+    SYSTEM_PROMPT += """
+<IMPORTANT>
+* Open applications using Spotlight by using the computer tool to simulate pressing Command+Space, typing the application name, and pressing Enter.
+</IMPORTANT>"""
+
 
 async def sampling_loop(
     *,
@@ -122,8 +134,8 @@ async def sampling_loop(
     """
     tool_collection = ToolCollection(
         ComputerTool(),
-        BashTool(),
-        EditTool(),
+        # BashTool(),
+        # EditTool(),
     )
     system = (
         f"{SYSTEM_PROMPT}{' ' + system_prompt_suffix if system_prompt_suffix else ''}"
@@ -227,7 +239,7 @@ async def sampling_loop(
 def _maybe_filter_to_n_most_recent_images(
     messages: list[BetaMessageParam],
     images_to_keep: int,
-    min_removal_threshold: int = 10,
+    min_removal_threshold: int = 5,
 ):
     """
     With the assumption that images are screenshots that are of diminishing value as
@@ -353,6 +365,9 @@ async def main():
 **Warning:** In this mode, Open Interpreter will not require approval before performing actions. Move your mouse to any corner of the screen to exit.
 """
 
+    markdown_text = f"""> Model set to `Claude 3.5`    OS control enabled
+"""
+
     print_markdown(markdown_text)
 
     # Start the mouse position checking thread

interpreter/computer_use/tools/computer.py

@@ -1,9 +1,12 @@
 import asyncio
 import base64
+import math
 import os
 import platform
 import shlex
 import shutil
+import tempfile
+import time
 from enum import StrEnum
 from pathlib import Path
 from typing import Literal, TypedDict
@@ -64,6 +67,30 @@ def chunks(s: str, chunk_size: int) -> list[str]:
     return [s[i : i + chunk_size] for i in range(0, len(s), chunk_size)]
 
 
+def smooth_move_to(x, y, duration=1.2):
+    start_x, start_y = pyautogui.position()
+    dx = x - start_x
+    dy = y - start_y
+    distance = math.hypot(dx, dy)  # Calculate the distance in pixels
+
+    start_time = time.time()
+
+    while True:
+        elapsed_time = time.time() - start_time
+        if elapsed_time > duration:
+            break
+
+        t = elapsed_time / duration
+        eased_t = (1 - math.cos(t * math.pi)) / 2  # easeInOutSine function
+
+        target_x = start_x + dx * eased_t
+        target_y = start_y + dy * eased_t
+        pyautogui.moveTo(target_x, target_y)
+
+    # Ensure the mouse ends up exactly at the target (x, y)
+    pyautogui.moveTo(x, y)
+
+
 class ComputerTool(BaseAnthropicTool):
     """
     A tool that allows the agent to interact with the screen, keyboard, and mouse of the current computer.
@@ -123,8 +150,9 @@ async def __call__(
             )
 
             if action == "mouse_move":
-                pyautogui.moveTo(x, y)
+                smooth_move_to(x, y)
             elif action == "left_click_drag":
+                smooth_move_to(x, y)
                 pyautogui.dragTo(x, y, button="left")
 
         elif action in ("key", "type"):
@@ -136,20 +164,38 @@ async def __call__(
                     text = text.replace("super+", "command+")
                 keys = text.split("+")
                 if len(keys) > 1:
-                    pyautogui.hotkey(*keys)
+                    if "darwin" in platform.system().lower():
+                        # Use AppleScript for hotkey on macOS
+                        keystroke, modifier = (keys[-1], "+".join(keys[:-1]))
+                        modifier = modifier.lower() + " down"
+                        if keystroke.lower() == "space":
+                            keystroke = " "
+                        elif keystroke.lower() == "enter":
+                            keystroke = "\n"
+                        script = f"""
+                        tell application "System Events"
+                            keystroke "{keystroke}" using {modifier}
+                        end tell
+                        """
+                        os.system("osascript -e '{}'".format(script))
+                    else:
+                        pyautogui.hotkey(*keys)
                 else:
                     pyautogui.press(text)
             elif action == "type":
                 pyautogui.write(text, interval=TYPING_DELAY_MS / 1000)
 
         elif action in ("left_click", "right_click", "double_click", "middle_click"):
+            time.sleep(0.1)
             button = {
                 "left_click": "left",
                 "right_click": "right",
                 "middle_click": "middle",
             }
             if action == "double_click":
-                pyautogui.doubleClick()
+                pyautogui.click()
+                time.sleep(0.1)
+                pyautogui.click()
             else:
                 pyautogui.click(button=button.get(action, "left"))
 
@@ -170,9 +216,9 @@ async def __call__(
 
     async def screenshot(self):
         """Take a screenshot of the current screen and return the base64 encoded image."""
-        output_dir = Path(OUTPUT_DIR)
-        output_dir.mkdir(parents=True, exist_ok=True)
-        path = output_dir / f"screenshot_{uuid4().hex}.png"
+        # Use a user-writable directory for temporary files
+        temp_dir = Path(tempfile.gettempdir())
+        path = temp_dir / f"screenshot_{uuid4().hex}.png"
 
         screenshot = pyautogui.screenshot()
         screenshot.save(str(path))
@@ -186,7 +232,9 @@ async def screenshot(self):
             )
 
         if path.exists():
-            return ToolResult(base64_image=base64.b64encode(path.read_bytes()).decode())
+            base64_image = base64.b64encode(path.read_bytes()).decode()
+            path.unlink()  # Remove the temporary file
+            return ToolResult(base64_image=base64_image)
         raise ToolError(f"Failed to take screenshot")
 
     async def shell(self, command: str, take_screenshot=True) -> ToolResult:

pyproject.toml

@@ -4,7 +4,7 @@ packages = [
     {include = "interpreter"},
     {include = "scripts"},
 ]
-version = "0.3.14" # Use "-rc1", "-rc2", etc. for pre-release versions
+version = "0.4.0" # Use "-rc1", "-rc2", etc. for pre-release versions
 description = "Let language models run code"
 authors = ["Killian Lucas <[email protected]>"]
 readme = "README.md"