Elements: locate

POST

api

elements:locate

Locate Element

curl --request POST \
  --url https://argus.axilio.ai/api/v1/elements:locate \
  --header 'Content-Type: application/json' \
  --header 'X-Axilio-Api-Key: <api-key>' \
  --data '
{
  "elements": [
    {
      "id": "<string>",
      "role": "<string>",
      "bounds": {
        "height": 1,
        "width": 1,
        "x": 123,
        "y": 123
      },
      "name": "",
      "platform": {},
      "value": "<string>"
    }
  ],
  "query": "<string>",
  "image": "<string>",
  "model": "<string>"
}
'

import requests

url = "https://argus.axilio.ai/api/v1/elements:locate"

payload = {
    "elements": [
        {
            "id": "<string>",
            "role": "<string>",
            "bounds": {
                "height": 1,
                "width": 1,
                "x": 123,
                "y": 123
            },
            "name": "",
            "platform": {},
            "value": "<string>"
        }
    ],
    "query": "<string>",
    "image": "<string>",
    "model": "<string>"
}
headers = {
    "X-Axilio-Api-Key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'X-Axilio-Api-Key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    elements: [
      {
        id: '<string>',
        role: '<string>',
        bounds: {height: 1, width: 1, x: 123, y: 123},
        name: '',
        platform: {},
        value: '<string>'
      }
    ],
    query: '<string>',
    image: '<string>',
    model: '<string>'
  })
};

fetch('https://argus.axilio.ai/api/v1/elements:locate', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://argus.axilio.ai/api/v1/elements:locate",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'elements' => [
        [
                'id' => '<string>',
                'role' => '<string>',
                'bounds' => [
                                'height' => 1,
                                'width' => 1,
                                'x' => 123,
                                'y' => 123
                ],
                'name' => '',
                'platform' => [
                                
                ],
                'value' => '<string>'
        ]
    ],
    'query' => '<string>',
    'image' => '<string>',
    'model' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "X-Axilio-Api-Key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://argus.axilio.ai/api/v1/elements:locate"

	payload := strings.NewReader("{\n  \"elements\": [\n    {\n      \"id\": \"<string>\",\n      \"role\": \"<string>\",\n      \"bounds\": {\n        \"height\": 1,\n        \"width\": 1,\n        \"x\": 123,\n        \"y\": 123\n      },\n      \"name\": \"\",\n      \"platform\": {},\n      \"value\": \"<string>\"\n    }\n  ],\n  \"query\": \"<string>\",\n  \"image\": \"<string>\",\n  \"model\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("X-Axilio-Api-Key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://argus.axilio.ai/api/v1/elements:locate")
  .header("X-Axilio-Api-Key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"elements\": [\n    {\n      \"id\": \"<string>\",\n      \"role\": \"<string>\",\n      \"bounds\": {\n        \"height\": 1,\n        \"width\": 1,\n        \"x\": 123,\n        \"y\": 123\n      },\n      \"name\": \"\",\n      \"platform\": {},\n      \"value\": \"<string>\"\n    }\n  ],\n  \"query\": \"<string>\",\n  \"image\": \"<string>\",\n  \"model\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://argus.axilio.ai/api/v1/elements:locate")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["X-Axilio-Api-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"elements\": [\n    {\n      \"id\": \"<string>\",\n      \"role\": \"<string>\",\n      \"bounds\": {\n        \"height\": 1,\n        \"width\": 1,\n        \"x\": 123,\n        \"y\": 123\n      },\n      \"name\": \"\",\n      \"platform\": {},\n      \"value\": \"<string>\"\n    }\n  ],\n  \"query\": \"<string>\",\n  \"image\": \"<string>\",\n  \"model\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "confidence": 0.5,
  "cost_microdollars": 1,
  "found": true,
  "latency_ms": 1,
  "model": "<string>",
  "completion_tokens": 0,
  "element_id": "<string>",
  "prompt_tokens": 0
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

Choose the UI element that a natural-language query describes from a list of elements you already have. You send the candidates, and Argus answers with one of your element id values as element_id, or found: false. It never answers with a point, so you act on the element itself.

Why elements and screenshots are separate operations

Screenshot grounding and element grounding take different inputs and give different answers. A screenshot locate reads pixels and returns a box. An element locate reads structured elements and returns an element ID. Keeping them as separate operations gives each one a request and response that fit it exactly. The element operation doesn’t care where your elements came from. A browser DOM, the results of Screenshots: detect, or any other UI tree all map onto the same shape: an id, a role, a name, an optional value, optional pixel bounds, and an optional platform object of native attributes, which Argus passes to the model as given.

Request

query describes the target, such as "the Sign in button". Up to 2,000 characters.
elements holds 1 to 2,000 candidates. Each id must be unique in the request; a duplicate is a validation error. An id is at most 256 characters, a role at most 64, a name or value at most 1,024, and a platform object at most 4 KiB as JSON.
image is an optional base64 PNG or JPEG screenshot. When you send one, give bounds in that screenshot’s pixel space.
model is an optional vision-language model from Models: list. Omit it to use the default.

{
  "query": "the Sign in button",
  "elements": [
    {"id": "12", "role": "textbox", "name": "Email", "value": ""},
    {"id": "13", "role": "textbox", "name": "Password"},
    {
      "id": "14",
      "role": "button",
      "name": "Sign in",
      "bounds": {"x": 96, "y": 1480, "width": 888, "height": 144}
    }
  ]
}

Response

found tells you whether the model chose an element. When it did, element_id is one of the IDs you sent. Argus checks the answer against your request, so an ID you didn’t send comes back as not found rather than as a guess. confidence is the model’s own estimate from 0 to 1.

{
  "found": true,
  "element_id": "14",
  "confidence": 0.97,
  "model": "anthropic/claude-haiku-4.5",
  "cost_microdollars": 412,
  "prompt_tokens": 1210,
  "completion_tokens": 18,
  "latency_ms": 640
}

Billing

Element locate is billed per token on the model it ran, the same way as Screenshots: locate. Usage rows label it with the endpoint element_locate. See Usage. Errors use the problem details shape.

Authorizations

X-Axilio-Api-Key

string

header

required

Axilio API key (axl_*).

Body

application/json

Argus POST /elements:locate request.

Grounds a natural-language query in a list of UI elements: the model picks one of elements. The screenshot is optional context; the answer is always an element, never a point.

elements

Element · object[]

required

Candidate elements

Required array length: 1 - 2000 elements

Show child attributes

query

string

required

Natural-language target description

Required string length: 1 - 2000

image

string | null

Optional base64 screenshot (PNG or JPEG) the element bounds are in

Maximum string length: 16777216

model

string | null

VLM model to use; must be one of the models from GET /models. Omit to use the server's configured default.

Minimum string length: 1

Response

Successful Response

Argus POST /elements:locate response.

element_id is one of the request's element ids when found, else null. It names the element rather than indexing the list, so the answer does not depend on element order. The billing fields match screenshot locate.

confidence

number

required

Model self-reported confidence

Required range: 0 <= x <= 1

cost_microdollars

integer

required

Provider-reported cost in microdollars

Required range: x >= 0

found

boolean

required

Whether the model picked an element

latency_ms

integer

required

End-to-end VLM call latency

Required range: x >= 0

model

string

required

Model identifier returned by the provider

completion_tokens

integer

default:0

Completion (output) tokens the model billed for this call

Required range: x >= 0

element_id

string | null

The chosen element's id; null when not found

prompt_tokens

integer

default:0

Prompt (input) tokens the model billed for this call

Required range: x >= 0

Locate Element

curl --request POST \
  --url https://argus.axilio.ai/api/v1/elements:locate \
  --header 'Content-Type: application/json' \
  --header 'X-Axilio-Api-Key: <api-key>' \
  --data '
{
  "elements": [
    {
      "id": "<string>",
      "role": "<string>",
      "bounds": {
        "height": 1,
        "width": 1,
        "x": 123,
        "y": 123
      },
      "name": "",
      "platform": {},
      "value": "<string>"
    }
  ],
  "query": "<string>",
  "image": "<string>",
  "model": "<string>"
}
'

import requests

url = "https://argus.axilio.ai/api/v1/elements:locate"

payload = {
    "elements": [
        {
            "id": "<string>",
            "role": "<string>",
            "bounds": {
                "height": 1,
                "width": 1,
                "x": 123,
                "y": 123
            },
            "name": "",
            "platform": {},
            "value": "<string>"
        }
    ],
    "query": "<string>",
    "image": "<string>",
    "model": "<string>"
}
headers = {
    "X-Axilio-Api-Key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'X-Axilio-Api-Key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    elements: [
      {
        id: '<string>',
        role: '<string>',
        bounds: {height: 1, width: 1, x: 123, y: 123},
        name: '',
        platform: {},
        value: '<string>'
      }
    ],
    query: '<string>',
    image: '<string>',
    model: '<string>'
  })
};

fetch('https://argus.axilio.ai/api/v1/elements:locate', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://argus.axilio.ai/api/v1/elements:locate",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'elements' => [
        [
                'id' => '<string>',
                'role' => '<string>',
                'bounds' => [
                                'height' => 1,
                                'width' => 1,
                                'x' => 123,
                                'y' => 123
                ],
                'name' => '',
                'platform' => [
                                
                ],
                'value' => '<string>'
        ]
    ],
    'query' => '<string>',
    'image' => '<string>',
    'model' => '<string>'
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "X-Axilio-Api-Key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://argus.axilio.ai/api/v1/elements:locate"

	payload := strings.NewReader("{\n  \"elements\": [\n    {\n      \"id\": \"<string>\",\n      \"role\": \"<string>\",\n      \"bounds\": {\n        \"height\": 1,\n        \"width\": 1,\n        \"x\": 123,\n        \"y\": 123\n      },\n      \"name\": \"\",\n      \"platform\": {},\n      \"value\": \"<string>\"\n    }\n  ],\n  \"query\": \"<string>\",\n  \"image\": \"<string>\",\n  \"model\": \"<string>\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("X-Axilio-Api-Key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://argus.axilio.ai/api/v1/elements:locate")
  .header("X-Axilio-Api-Key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"elements\": [\n    {\n      \"id\": \"<string>\",\n      \"role\": \"<string>\",\n      \"bounds\": {\n        \"height\": 1,\n        \"width\": 1,\n        \"x\": 123,\n        \"y\": 123\n      },\n      \"name\": \"\",\n      \"platform\": {},\n      \"value\": \"<string>\"\n    }\n  ],\n  \"query\": \"<string>\",\n  \"image\": \"<string>\",\n  \"model\": \"<string>\"\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://argus.axilio.ai/api/v1/elements:locate")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["X-Axilio-Api-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"elements\": [\n    {\n      \"id\": \"<string>\",\n      \"role\": \"<string>\",\n      \"bounds\": {\n        \"height\": 1,\n        \"width\": 1,\n        \"x\": 123,\n        \"y\": 123\n      },\n      \"name\": \"\",\n      \"platform\": {},\n      \"value\": \"<string>\"\n    }\n  ],\n  \"query\": \"<string>\",\n  \"image\": \"<string>\",\n  \"model\": \"<string>\"\n}"

response = http.request(request)
puts response.read_body

{
  "confidence": 0.5,
  "cost_microdollars": 1,
  "found": true,
  "latency_ms": 1,
  "model": "<string>",
  "completion_tokens": 0,
  "element_id": "<string>",
  "prompt_tokens": 0
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

Overview

Device Control Protocol

REST API

Argus API

Elements: locate

Why elements and screenshots are separate operations

Request

Response

Billing

Authorizations

Body

Response

​Why elements and screenshots are separate operations

​Request

​Response

​Billing

Authorizations

Body

Response

Why elements and screenshots are separate operations

Request

Response

Billing