AI SDK

import { screen, contentMatchingDescription } from "@nut-tree/nut-js";
import { useOpenAIVisionProvider } from "@nut-tree/plugin-ai-sdk";

// Picks up OPENAI_API_KEY from the environment
useOpenAIVisionProvider({ model: "gpt-5-mini" });

// Find elements by description
const region = await screen.find(
    contentMatchingDescription("the login button")
);

import { screen, contentMatchingDescription } from "@nut-tree/nut-js";
import { useAnthropicVisionProvider } from "@nut-tree/plugin-ai-sdk";

// Picks up ANTHROPIC_API_KEY from the environment
useAnthropicVisionProvider({ model: "claude-sonnet-5" });

const region = await screen.find(
    contentMatchingDescription("the submit button")
);

import { screen, contentMatchingDescription } from "@nut-tree/nut-js";
import { useOllamaVisionProvider } from "@nut-tree/plugin-ai-sdk";

useOllamaVisionProvider({
    model: "llama3.2-vision:11b",
    ollama: { baseURL: "http://localhost:11434/api" },
});

const region = await screen.find(
    contentMatchingDescription("the search input field")
);

import { screen, contentMatchingDescription } from "@nut-tree/nut-js";
import { useOpenAICompatibleVisionProvider } from "@nut-tree/plugin-ai-sdk";

useOpenAICompatibleVisionProvider({
    model: "Qwen/Qwen3-VL-8B-Instruct",
    openaiCompatible: {
        baseURL: "http://localhost:8000/v1",
    },
});

const region = await screen.find(
    contentMatchingDescription("the 'Sign in' button")
);

useOpenAICompatibleVisionProvider({
    model: "google/gemma-3-12b-it",
    openaiCompatible: { baseURL: "http://localhost:1234/v1" },
    coordinateSpace: "normalized",
    coordinateScale: 1000,
});

const region = await screen.find(
    contentMatchingDescription("the 'Export' button"), {
        providerData: {
            model: { provider: "openai-compatible", model: "Qwen/Qwen3-VL-8B-Instruct" },
            maxMatches: 3,
        },
    }
);

interface VercelAiSdkVisionProviderConfig {
  defaultModel?: { provider: "openai" | "anthropic" | "ollama" | "openai-compatible"; model: string };
  defaultConfidence?: number;        // [0, 1], default 0.8
  defaultMaxMatches?: number;
  coordinateSpace?: "pixels" | "normalized";   // new in 2.0, default "pixels"
  coordinateScale?: number;                    // new in 2.0, default 1000
  openai?: { apiKey?: string; baseURL?: string };
  anthropic?: { apiKey?: string; baseURL?: string };
  ollama?: { baseURL?: string };
  openaiCompatible?: {                         // new in 2.0
    baseURL: string;
    apiKey?: string;
    name?: string;
    headers?: Record<string, string>;
    queryParams?: Record<string, string>;
    supportsStructuredOutputs?: boolean;
  };
}

import { screen, mouse, centerOf, straightTo, Button, contentMatchingDescription } from "@nut-tree/nut-js";
import { useOpenAIVisionProvider } from "@nut-tree/plugin-ai-sdk";

useOpenAIVisionProvider({ model: "gpt-5-mini" });

// Find a specific UI element by description
const region = await screen.find(
    contentMatchingDescription("a system menu called 'Navigate'"), {
        confidence: 0.8,
    }
);

// Interact with the found region
await mouse.move(straightTo(centerOf(region)));
await mouse.click(Button.LEFT);

// Wait up to 10 seconds for a dialog to appear, checking every second
const dialog = await screen.waitFor(
    contentMatchingDescription("a confirmation dialog with 'Save changes?' text"),
    10000,
    1000
);

import { screen, contentMatchingDescription } from "@nut-tree/nut-js";
import { useOpenAIVisionProvider } from "@nut-tree/plugin-ai-sdk";

useOpenAIVisionProvider({ model: "gpt-5-mini" });

// Assert that the screen shows specific content
await expect(screen).toShow(
    contentMatchingDescription("a browser window with an open npmjs.org tab")
);

// Assert with a custom confidence threshold
await expect(screen).toShow(
    contentMatchingDescription("a navigation bar with a 'Home' link"),
    { confidence: 0.9 }
);

import { describe, it, expect, beforeAll } from "vitest";
import { screen, mouse, keyboard, centerOf, straightTo, Button, contentMatchingDescription } from "@nut-tree/nut-js";
import { useOpenAIVisionProvider } from "@nut-tree/plugin-ai-sdk";

beforeAll(() => {
    useOpenAIVisionProvider({ model: "gpt-5-mini" });
});

describe("Application navigation", () => {
    it("should open the settings page", async () => {
        // Find and click the settings menu
        const settingsMenu = await screen.find(
            contentMatchingDescription("a menu item labeled 'Settings'")
        );
        await mouse.move(straightTo(centerOf(settingsMenu)));
        await mouse.click(Button.LEFT);

        // Verify the settings page is displayed
        await expect(screen).toShow(
            contentMatchingDescription("a settings page with 'General' and 'Advanced' tabs")
        );
    });

    it("should display search results", async () => {
        await keyboard.type("nut.js automation");

        await expect(screen).toShow(
            contentMatchingDescription("a list of search results related to 'nut.js'")
        );
    });
});