Screenshots: locate

POST

api

screenshots:locate

Locate In Screenshot

curl --request POST \
  --url https://argus.axilio.ai/api/v1/screenshots:locate \
  --header 'Content-Type: application/json' \
  --header 'X-Axilio-Api-Key: <api-key>' \
  --data '
{
  "image": "<string>",
  "query": "<string>",
  "model": "<string>",
  "texts": [
    {
      "bbox": {
        "x1": 123,
        "x2": 123,
        "y1": 123,
        "y2": 123
      },
      "text": "<string>"
    }
  ]
}
'

import requests

url = "https://argus.axilio.ai/api/v1/screenshots:locate"

payload = {
    "image": "<string>",
    "query": "<string>",
    "model": "<string>",
    "texts": [
        {
            "bbox": {
                "x1": 123,
                "x2": 123,
                "y1": 123,
                "y2": 123
            },
            "text": "<string>"
        }
    ]
}
headers = {
    "X-Axilio-Api-Key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'X-Axilio-Api-Key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    image: '<string>',
    query: '<string>',
    model: '<string>',
    texts: [{bbox: {x1: 123, x2: 123, y1: 123, y2: 123}, text: '<string>'}]
  })
};

fetch('https://argus.axilio.ai/api/v1/screenshots:locate', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://argus.axilio.ai/api/v1/screenshots:locate",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'image' => '<string>',
    'query' => '<string>',
    'model' => '<string>',
    'texts' => [
        [
                'bbox' => [
                                'x1' => 123,
                                'x2' => 123,
                                'y1' => 123,
                                'y2' => 123
                ],
                'text' => '<string>'
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "X-Axilio-Api-Key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://argus.axilio.ai/api/v1/screenshots:locate"

	payload := strings.NewReader("{\n  \"image\": \"<string>\",\n  \"query\": \"<string>\",\n  \"model\": \"<string>\",\n  \"texts\": [\n    {\n      \"bbox\": {\n        \"x1\": 123,\n        \"x2\": 123,\n        \"y1\": 123,\n        \"y2\": 123\n      },\n      \"text\": \"<string>\"\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("X-Axilio-Api-Key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://argus.axilio.ai/api/v1/screenshots:locate")
  .header("X-Axilio-Api-Key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"image\": \"<string>\",\n  \"query\": \"<string>\",\n  \"model\": \"<string>\",\n  \"texts\": [\n    {\n      \"bbox\": {\n        \"x1\": 123,\n        \"x2\": 123,\n        \"y1\": 123,\n        \"y2\": 123\n      },\n      \"text\": \"<string>\"\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://argus.axilio.ai/api/v1/screenshots:locate")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["X-Axilio-Api-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"image\": \"<string>\",\n  \"query\": \"<string>\",\n  \"model\": \"<string>\",\n  \"texts\": [\n    {\n      \"bbox\": {\n        \"x1\": 123,\n        \"x2\": 123,\n        \"y1\": 123,\n        \"y2\": 123\n      },\n      \"text\": \"<string>\"\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "confidence": 0.5,
  "cost_microdollars": 1,
  "found": true,
  "latency_ms": 1,
  "model": "<string>",
  "bbox": {
    "height": 123,
    "width": 123,
    "x": 1,
    "y": 1
  },
  "completion_tokens": 0,
  "matched_text_index": 1,
  "prompt_tokens": 0
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

Find the element a natural-language query describes, such as "the blue Continue button", in a screenshot you supply. A vision-language model reads the image and answers in one of two ways when found is true:

A box. bbox is the element’s pixel box in the screenshot.
A matched text. When you send texts, the OCR results for the same screenshot, the model can answer with matched_text_index instead. It points into your texts list, so you reuse that entry’s exact box.

Argus does not run OCR for this operation. To ground answers in text, call Screenshots: detect first and pass its text results as texts unchanged: detect already returns each text box normalized to [0, 1] of the image, which is the form texts takes. Only the answer’s bbox is in pixels. Pass a model from Models: list, or omit it to use the default. The call is billed per token on that model, and the response reports prompt_tokens, completion_tokens, and cost_microdollars. If you already have the screen’s UI elements, for example from a DOM, use Elements: locate instead. It answers with one of your elements, which is exact where a box is approximate. Errors use the problem details shape.

Authorizations

X-Axilio-Api-Key

string

header

required

Axilio API key (axl_*).

Body

application/json

Argus POST /screenshots:locate request.

The caller pre-computes OCR via POST /screenshots:detect (or reuses cached results keyed by frame hash) and passes the text list here. Argus does NOT re-run OCR; the VLM call uses the provided texts as grounding context.

image

string

required

Base64 encoded image (PNG or JPEG)

Maximum string length: 16777216

query

string

required

Natural-language target description

Minimum string length: 1

model

string | null

VLM model to use; must be one of the models from GET /models. Omit to use the server's configured default. The system prompt is fixed to the element-locator task.

Minimum string length: 1

texts

TextElementInput · object[]

Pre-computed OCR text elements. Empty list means Argus skips OCR grounding and asks the VLM to locate from the image alone.

Show child attributes

Response

Successful Response

Argus POST /screenshots:locate response.

Exactly one of matched_text_index or bbox is populated when found is True. matched_text_index indicates the VLM grounded its answer to a specific OCR text element (pixel-accurate bbox available via the original texts list). bbox is a free-form VLM bbox.

confidence

number

required

Model self-reported confidence

Required range: 0 <= x <= 1

cost_microdollars

integer

required

Provider-reported cost in microdollars

Required range: x >= 0

found

boolean

required

Whether the VLM located the target

latency_ms

integer

required

End-to-end VLM call latency

Required range: x >= 0

model

string

required

Model identifier returned by the provider

bbox

LocateBBox · object | null

Pixel-space bbox. None when matched_text_index is set or found=False.

Show child attributes

completion_tokens

integer

default:0

Completion (output) tokens the model billed for this call

Required range: x >= 0

matched_text_index

integer | null

Index into the request texts list when the answer is OCR-grounded.

Required range: x >= 0

prompt_tokens

integer

default:0

Prompt (input) tokens the model billed for this call

Required range: x >= 0

Locate In Screenshot

curl --request POST \
  --url https://argus.axilio.ai/api/v1/screenshots:locate \
  --header 'Content-Type: application/json' \
  --header 'X-Axilio-Api-Key: <api-key>' \
  --data '
{
  "image": "<string>",
  "query": "<string>",
  "model": "<string>",
  "texts": [
    {
      "bbox": {
        "x1": 123,
        "x2": 123,
        "y1": 123,
        "y2": 123
      },
      "text": "<string>"
    }
  ]
}
'

import requests

url = "https://argus.axilio.ai/api/v1/screenshots:locate"

payload = {
    "image": "<string>",
    "query": "<string>",
    "model": "<string>",
    "texts": [
        {
            "bbox": {
                "x1": 123,
                "x2": 123,
                "y1": 123,
                "y2": 123
            },
            "text": "<string>"
        }
    ]
}
headers = {
    "X-Axilio-Api-Key": "<api-key>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {'X-Axilio-Api-Key': '<api-key>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    image: '<string>',
    query: '<string>',
    model: '<string>',
    texts: [{bbox: {x1: 123, x2: 123, y1: 123, y2: 123}, text: '<string>'}]
  })
};

fetch('https://argus.axilio.ai/api/v1/screenshots:locate', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://argus.axilio.ai/api/v1/screenshots:locate",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'image' => '<string>',
    'query' => '<string>',
    'model' => '<string>',
    'texts' => [
        [
                'bbox' => [
                                'x1' => 123,
                                'x2' => 123,
                                'y1' => 123,
                                'y2' => 123
                ],
                'text' => '<string>'
        ]
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "X-Axilio-Api-Key: <api-key>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://argus.axilio.ai/api/v1/screenshots:locate"

	payload := strings.NewReader("{\n  \"image\": \"<string>\",\n  \"query\": \"<string>\",\n  \"model\": \"<string>\",\n  \"texts\": [\n    {\n      \"bbox\": {\n        \"x1\": 123,\n        \"x2\": 123,\n        \"y1\": 123,\n        \"y2\": 123\n      },\n      \"text\": \"<string>\"\n    }\n  ]\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("X-Axilio-Api-Key", "<api-key>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://argus.axilio.ai/api/v1/screenshots:locate")
  .header("X-Axilio-Api-Key", "<api-key>")
  .header("Content-Type", "application/json")
  .body("{\n  \"image\": \"<string>\",\n  \"query\": \"<string>\",\n  \"model\": \"<string>\",\n  \"texts\": [\n    {\n      \"bbox\": {\n        \"x1\": 123,\n        \"x2\": 123,\n        \"y1\": 123,\n        \"y2\": 123\n      },\n      \"text\": \"<string>\"\n    }\n  ]\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://argus.axilio.ai/api/v1/screenshots:locate")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["X-Axilio-Api-Key"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"image\": \"<string>\",\n  \"query\": \"<string>\",\n  \"model\": \"<string>\",\n  \"texts\": [\n    {\n      \"bbox\": {\n        \"x1\": 123,\n        \"x2\": 123,\n        \"y1\": 123,\n        \"y2\": 123\n      },\n      \"text\": \"<string>\"\n    }\n  ]\n}"

response = http.request(request)
puts response.read_body

{
  "confidence": 0.5,
  "cost_microdollars": 1,
  "found": true,
  "latency_ms": 1,
  "model": "<string>",
  "bbox": {
    "height": 123,
    "width": 123,
    "x": 1,
    "y": 1
  },
  "completion_tokens": 0,
  "matched_text_index": 1,
  "prompt_tokens": 0
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

{
  "code": "<string>",
  "detail": "<string>",
  "status": 123,
  "title": "<string>",
  "errors": [
    {}
  ],
  "type": "about:blank"
}

Overview

Device Control Protocol

REST API

Argus API

Screenshots: locate

Authorizations

Body

Response