> ## Documentation Index
> Fetch the complete documentation index at: https://docs.bytebot.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Computer Action

> Execute computer actions in the virtual desktop environment

Execute actions like mouse movements, clicks, keyboard input, and screenshots in the Bytebot desktop environment.

## Request

<ParamField body="action" type="string" required>
  The type of computer action to perform. Must be one of: `move_mouse`, `trace_mouse`,
  `click_mouse`, `press_mouse`, `drag_mouse`, `scroll`, `type_keys`, `press_keys`,
  `type_text`, `wait`, `screenshot`, `cursor_position`.
</ParamField>

### Mouse Actions

<Accordion title="move_mouse">
  <ParamField body="coordinates" type="object" required>
    The target coordinates to move to.

    <Expandable title="coordinates properties">
      <ParamField body="x" type="number" required>
        X coordinate (horizontal position)
      </ParamField>

      <ParamField body="y" type="number" required>
        Y coordinate (vertical position)
      </ParamField>
    </Expandable>
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "move_mouse",
    "coordinates": {
      "x": 100,
      "y": 200
    }
  }
  ```
</Accordion>

<Accordion title="trace_mouse">
  <ParamField body="path" type="array" required>
    Array of coordinate objects for the mouse path.

    <Expandable title="path">
      <ParamField body="0" type="object">
        <Expandable title="properties">
          <ParamField body="x" type="number" required>
            X coordinate
          </ParamField>

          <ParamField body="y" type="number" required>
            Y coordinate
          </ParamField>
        </Expandable>
      </ParamField>
    </Expandable>
  </ParamField>

  {" "}

  <ParamField body="holdKeys" type="array">
    Keys to hold while moving the mouse along the path.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "trace_mouse",
    "path": [
      { "x": 100, "y": 100 },
      { "x": 150, "y": 150 },
      { "x": 200, "y": 200 }
    ],
    "holdKeys": ["shift"]
  }
  ```
</Accordion>

<Accordion title="click_mouse">
  <ParamField body="coordinates" type="object">
    The coordinates to click (uses current cursor position if omitted).

    <Expandable title="coordinates properties">
      <ParamField body="x" type="number" required>
        X coordinate (horizontal position)
      </ParamField>

      <ParamField body="y" type="number" required>
        Y coordinate (vertical position)
      </ParamField>
    </Expandable>
  </ParamField>

  {" "}

  <ParamField body="button" type="string" required>
    Mouse button to click. Must be one of: `left`, `right`, `middle`.
  </ParamField>

  {" "}

  <ParamField body="clickCount" type="number" required>
    Number of clicks to perform.
  </ParamField>

  {" "}

  <ParamField body="holdKeys" type="array">
    Keys to hold while clicking (e.g., \['ctrl', 'shift'])

    <Expandable title="holdKeys">
      <ParamField body="0" type="string">
        Key name
      </ParamField>
    </Expandable>
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "click_mouse",
    "coordinates": {
      "x": 150,
      "y": 250
    },
    "button": "left",
    "clickCount": 2
  }
  ```
</Accordion>

<Accordion title="press_mouse">
  <ParamField body="coordinates" type="object">
    The coordinates to press/release (uses current cursor position if omitted).

    <Expandable title="coordinates properties">
      <ParamField body="x" type="number" required>
        X coordinate (horizontal position)
      </ParamField>

      <ParamField body="y" type="number" required>
        Y coordinate (vertical position)
      </ParamField>
    </Expandable>
  </ParamField>

  {" "}

  <ParamField body="button" type="string" required>
    Mouse button to press/release. Must be one of: `left`, `right`, `middle`.
  </ParamField>

  {" "}

  <ParamField body="press" type="string" required>
    Whether to press or release the button. Must be one of: `up`, `down`.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "press_mouse",
    "coordinates": {
      "x": 150,
      "y": 250
    },
    "button": "left",
    "press": "down"
  }
  ```
</Accordion>

<Accordion title="drag_mouse">
  <ParamField body="path" type="array" required>
    Array of coordinate objects for the drag path.

    <Expandable title="path">
      <ParamField body="0" type="object">
        <Expandable title="properties">
          <ParamField body="x" type="number" required>
            X coordinate
          </ParamField>

          <ParamField body="y" type="number" required>
            Y coordinate
          </ParamField>
        </Expandable>
      </ParamField>
    </Expandable>
  </ParamField>

  {" "}

  <ParamField body="button" type="string" required>
    Mouse button to use for dragging. Must be one of: `left`, `right`, `middle`.
  </ParamField>

  {" "}

  <ParamField body="holdKeys" type="array">
    Keys to hold while dragging.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "drag_mouse",
    "path": [
      { "x": 100, "y": 100 },
      { "x": 200, "y": 200 }
    ],
    "button": "left"
  }
  ```
</Accordion>

<Accordion title="scroll">
  <ParamField body="coordinates" type="object">
    The coordinates to scroll at (uses current cursor position if omitted).

    <Expandable title="coordinates properties">
      <ParamField body="x" type="number" required>
        X coordinate
      </ParamField>

      <ParamField body="y" type="number" required>
        Y coordinate
      </ParamField>
    </Expandable>
  </ParamField>

  {" "}

  <ParamField body="direction" type="string" required>
    Scroll direction. Must be one of: `up`, `down`, `left`, `right`.
  </ParamField>

  {" "}

  <ParamField body="scrollCount" type="number" required>
    Number of scroll steps to perform.
  </ParamField>

  {" "}

  <ParamField body="holdKeys" type="array">
    Keys to hold while scrolling.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "scroll",
    "direction": "down",
    "scrollCount": 5
  }
  ```
</Accordion>

### Keyboard Actions

<Accordion title="type_keys">
  <ParamField body="keys" type="array" required>
    Array of keys to type in sequence.

    <Expandable title="keys">
      <ParamField body="0" type="string">
        Key name
      </ParamField>
    </Expandable>
  </ParamField>

  {" "}

  <ParamField body="delay" type="number">
    Delay between key presses in milliseconds.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "type_keys",
    "keys": ["a", "b", "c", "enter"],
    "delay": 50
  }
  ```
</Accordion>

<Accordion title="press_keys">
  <ParamField body="keys" type="array" required>
    Array of keys to press or release.

    <Expandable title="keys">
      <ParamField body="0" type="string">
        Key name
      </ParamField>
    </Expandable>
  </ParamField>

  {" "}

  <ParamField body="press" type="string" required>
    Whether to press or release the keys. Must be one of: `up`, `down`.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "press_keys",
    "keys": ["ctrl", "shift", "esc"],
    "press": "down"
  }
  ```
</Accordion>

<Accordion title="type_text">
  <ParamField body="text" type="string" required>
    The text string to type.
  </ParamField>

  {" "}

  <ParamField body="delay" type="number">
    Delay between characters in milliseconds.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "type_text",
    "text": "Hello, Bytebot!",
    "delay": 50
  }
  ```
</Accordion>

<Accordion title="paste_text">
  <ParamField body="text" type="string" required>
    The text to paste. Useful for special characters that aren't on the standard keyboard.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "paste_text",
    "text": "Special characters: ©®™€¥£ émojis 🎉"
  }
  ```
</Accordion>

### System Actions

<Accordion title="wait">
  <ParamField body="duration" type="number" required>
    Wait duration in milliseconds.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "wait",
    "duration": 2000
  }
  ```
</Accordion>

<Accordion title="screenshot">
  No parameters required.

  **Example Request**

  ```json theme={null}
  {
    "action": "screenshot"
  }
  ```
</Accordion>

<Accordion title="cursor_position">
  No parameters required.

  **Example Request**

  ```json theme={null}
  {
    "action": "cursor_position"
  }
  ```
</Accordion>

<Accordion title="application">
  <ParamField body="application" type="string" required>
    The application to switch to. Available options: `firefox`, `1password`, `thunderbird`, `vscode`, `terminal`, `desktop`, `directory`.
  </ParamField>

  **Example Request**

  ```json theme={null}
  {
    "action": "application",
    "application": "firefox"
  }
  ```

  **Available Applications:**

  * `firefox` - Mozilla Firefox web browser
  * `1password` - Password manager
  * `thunderbird` - Email client
  * `vscode` - Visual Studio Code editor
  * `terminal` - Terminal/console application
  * `desktop` - Switch to desktop
  * `directory` - File manager/directory browser
</Accordion>

## Response

Responses vary based on the action performed:

### Default Response

Most actions return a simple success response:

```json theme={null}
{
  "success": true
}
```

### Screenshot Response

Returns the screenshot as a base64 encoded string:

```json theme={null}
{
  "success": true,
  "data": {
    "image": "base64_encoded_image_data"
  }
}
```

### Cursor Position Response

Returns the current cursor position:

```json theme={null}
{
  "success": true,
  "data": {
    "x": 123,
    "y": 456
  }
}
```

### Error Response

```json theme={null}
{
  "success": false,
  "error": "Error message"
}
```

### Code Examples

<CodeGroup>
  ```bash cURL theme={null}
  curl -X POST http://localhost:9990/computer-use \
    -H "Content-Type: application/json" \
    -d '{"action": "move_mouse", "coordinates": {"x": 100, "y": 200}}'
  ```

  ```python Python theme={null}
  import requests

  def control_computer(action, **params):
      url = "http://localhost:9990/computer-use"
      data = {"action": action, **params}
      response = requests.post(url, json=data)
      return response.json()

  # Move the mouse
  result = control_computer("move_mouse", coordinates={"x": 100, "y": 100})
  print(result)
  ```

  ```javascript JavaScript theme={null}
  const axios = require("axios");

  async function controlComputer(action, params = {}) {
    const url = "http://localhost:9990/computer-use";
    const data = { action, ...params };
    const response = await axios.post(url, data);
    return response.data;
  }

  // Move mouse example
  controlComputer("move_mouse", { coordinates: { x: 100, y: 100 } })
    .then((result) => console.log(result))
    .catch((error) => console.error("Error:", error));
  ```
</CodeGroup>


## OpenAPI

````yaml POST /computer-use
openapi: 3.1.0
info:
  title: Bytebot Computer Use API
  version: 1.0.0
  description: Control the Bytebot virtual desktop via a single endpoint
servers: []
security: []
paths:
  /computer-use:
    post:
      summary: Execute a computer action
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ComputerAction'
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ComputerActionResponse'
        '500':
          description: Error executing action
          content:
            application/json:
              schema:
                type: object
                properties:
                  status:
                    type: string
                  error:
                    type: string
components:
  schemas:
    ComputerAction:
      oneOf:
        - $ref: '#/components/schemas/MoveMouseAction'
        - $ref: '#/components/schemas/TraceMouseAction'
        - $ref: '#/components/schemas/ClickMouseAction'
        - $ref: '#/components/schemas/PressMouseAction'
        - $ref: '#/components/schemas/DragMouseAction'
        - $ref: '#/components/schemas/ScrollAction'
        - $ref: '#/components/schemas/TypeKeysAction'
        - $ref: '#/components/schemas/PressKeysAction'
        - $ref: '#/components/schemas/TypeTextAction'
        - $ref: '#/components/schemas/WaitAction'
        - $ref: '#/components/schemas/ScreenshotAction'
        - $ref: '#/components/schemas/CursorPositionAction'
      discriminator:
        propertyName: action
        mapping:
          move_mouse: '#/components/schemas/MoveMouseAction'
          trace_mouse: '#/components/schemas/TraceMouseAction'
          click_mouse: '#/components/schemas/ClickMouseAction'
          press_mouse: '#/components/schemas/PressMouseAction'
          drag_mouse: '#/components/schemas/DragMouseAction'
          scroll: '#/components/schemas/ScrollAction'
          type_keys: '#/components/schemas/TypeKeysAction'
          press_keys: '#/components/schemas/PressKeysAction'
          type_text: '#/components/schemas/TypeTextAction'
          wait: '#/components/schemas/WaitAction'
          screenshot: '#/components/schemas/ScreenshotAction'
          cursor_position: '#/components/schemas/CursorPositionAction'
    ComputerActionResponse:
      type: object
      properties:
        success:
          type: boolean
        data:
          oneOf:
            - $ref: '#/components/schemas/ScreenshotResponse'
            - $ref: '#/components/schemas/CursorPosition'
      required:
        - success
    MoveMouseAction:
      type: object
      properties:
        action:
          enum:
            - move_mouse
        coordinates:
          $ref: '#/components/schemas/Coordinates'
      required:
        - action
        - coordinates
    TraceMouseAction:
      type: object
      properties:
        action:
          enum:
            - trace_mouse
        path:
          type: array
          items:
            $ref: '#/components/schemas/Coordinates'
        holdKeys:
          type: array
          items:
            type: string
      required:
        - action
        - path
    ClickMouseAction:
      type: object
      properties:
        action:
          enum:
            - click_mouse
        coordinates:
          $ref: '#/components/schemas/Coordinates'
        button:
          $ref: '#/components/schemas/Button'
        holdKeys:
          type: array
          items:
            type: string
        clickCount:
          type: integer
          minimum: 1
      required:
        - action
        - button
        - clickCount
    PressMouseAction:
      type: object
      properties:
        action:
          enum:
            - press_mouse
        coordinates:
          $ref: '#/components/schemas/Coordinates'
        button:
          $ref: '#/components/schemas/Button'
        press:
          $ref: '#/components/schemas/Press'
      required:
        - action
        - button
        - press
    DragMouseAction:
      type: object
      properties:
        action:
          enum:
            - drag_mouse
        path:
          type: array
          items:
            $ref: '#/components/schemas/Coordinates'
        button:
          $ref: '#/components/schemas/Button'
        holdKeys:
          type: array
          items:
            type: string
      required:
        - action
        - path
        - button
    ScrollAction:
      type: object
      properties:
        action:
          enum:
            - scroll
        coordinates:
          $ref: '#/components/schemas/Coordinates'
        direction:
          $ref: '#/components/schemas/ScrollDirection'
        scrollCount:
          type: integer
          minimum: 1
        holdKeys:
          type: array
          items:
            type: string
      required:
        - action
        - direction
        - scrollCount
    TypeKeysAction:
      type: object
      properties:
        action:
          enum:
            - type_keys
        keys:
          type: array
          items:
            type: string
        delay:
          type: integer
          minimum: 0
      required:
        - action
        - keys
    PressKeysAction:
      type: object
      properties:
        action:
          enum:
            - press_keys
        keys:
          type: array
          items:
            type: string
        press:
          $ref: '#/components/schemas/Press'
      required:
        - action
        - keys
        - press
    TypeTextAction:
      type: object
      properties:
        action:
          enum:
            - type_text
        text:
          type: string
        delay:
          type: integer
          minimum: 0
      required:
        - action
        - text
    WaitAction:
      type: object
      properties:
        action:
          enum:
            - wait
        duration:
          type: integer
          minimum: 0
      required:
        - action
        - duration
    ScreenshotAction:
      type: object
      properties:
        action:
          enum:
            - screenshot
      required:
        - action
    CursorPositionAction:
      type: object
      properties:
        action:
          enum:
            - cursor_position
      required:
        - action
    ScreenshotResponse:
      type: object
      properties:
        image:
          type: string
          description: Base64 encoded PNG
      required:
        - image
    CursorPosition:
      type: object
      properties:
        x:
          type: number
        'y':
          type: number
      required:
        - x
        - 'y'
    Coordinates:
      type: object
      properties:
        x:
          type: number
        'y':
          type: number
      required:
        - x
        - 'y'
    Button:
      type: string
      enum:
        - left
        - right
        - middle
    Press:
      type: string
      enum:
        - up
        - down
    ScrollDirection:
      type: string
      enum:
        - up
        - down
        - left
        - right

````