@nut-tree/selenium-bridge

Installation

npm i @nut-tree/selenium-bridge

Buy

@nut-tree/selenium-bridge is included in the Solo and Team plans, as well as the OCR and nl-matcher packages.

Description

@nut-tree/selenium-bridge is not a classical nut.js plugin like e.g. @nut-tree/nl-matcher. It is a context provider that allows you to use Selenium in your nut.js scripts.

By passing a WebDriver instance to the Selenium bridge, nut.js can interact with the browser controlled by Selenium. This allows you to switch the nut.js context from your whole desktop to just the browser viewport, making it easy to use the nut.js image search or OCR features in the context of your Selenium tests.

Usage

Simply require / import the package to wire up the provider using either useSelenium:

import {Browser, Builder} from "selenium-webdriver";
import {Options} from "selenium-webdriver/chrome.js";
import {Button, centerOf, imageResource, keyboard, mouse, screen} from "@nut-tree/nut-js";
import "@nut-tree/nl-matcher";
import {useSelenium} from "@nut-tree/selenium-bridge";

let driver;

const setup = async () => {
    const options = new Options();

    screen.config.autoHighlight = true;
    keyboard.config.autoDelayMs = 10;
    screen.config.resourceDirectory = '../non-headless';

    let driver;
    console.log("Running locally");
    if (process.env.RUN_HEADLESS) {
        driver = await new Builder()
            .forBrowser(Browser.CHROME)
            .setChromeOptions(options.addArguments("--headless=new"))
            .build();
    } else {
        driver = await new Builder().forBrowser(Browser.CHROME).build();
    }
    return driver;
}

try {
    driver = await setup();

    useSelenium({driver});

    await driver.manage().setTimeouts({implicit: 500});
    await driver.get('https://google.com');

    const acceptButton = await screen.find(imageResource('target.png'), {
        confidence: 0.95,
        providerData: {validateMatches: true}
    });
    await mouse.move(centerOf(acceptButton));
    await mouse.click(Button.LEFT);

} catch (e) {
    console.error(e);
} finally {
    await driver.quit();
}

Configuration

Currently, @nut-tree/selenium-bridge does not require any configuration.

Working with WebElements

@nut-tree/selenium-bridge also provides a function that allows you to locate elements on a page by their position. This is useful when you want to interact with elements on a page that you can't easily locate by their attributes.

You simply pass the result of a screen.find call to the elementAt function, and it will return a Selenium WebElement.

Let's revisit the previous example and see how it would look like with the custom locator:

import {Browser, Builder} from "selenium-webdriver";
import {Options} from "selenium-webdriver/chrome.js";
import {imageResource, keyboard, screen} from "@nut-tree/nut-js";
import "@nut-tree/nl-matcher";
import {elementAt, useSelenium} from "@nut-tree/selenium-bridge";

let driver;

const setup = async () => {
    const options = new Options();

    screen.config.autoHighlight = true;
    keyboard.config.autoDelayMs = 10;
    screen.config.resourceDirectory = '../non-headless';

    let driver;
    console.log("Running locally");
    if (process.env.RUN_HEADLESS) {
        driver = await new Builder()
            .forBrowser(Browser.CHROME)
            .setChromeOptions(options.addArguments("--headless=new"))
            .build();
    } else {
        driver = await new Builder().forBrowser(Browser.CHROME).build();
    }
    return driver;
}

try {
    driver = await setup();

    useSelenium({driver});

    await driver.manage().setTimeouts({implicit: 500});
    await driver.get('https://google.com');

    const acceptButton = await screen.find(imageResource('target.png'), {
        confidence: 0.95,
        providerData: {validateMatches: true}
    });
    const acceptButtonElement = await elementAt(acceptButton);
    await acceptButtonElement.click();
} catch (e) {
    console.error(e);
} finally {
    await driver.quit();
}

Usage example

import {Browser, Builder} from "selenium-webdriver";
import {Options} from "selenium-webdriver/chrome.js";
import {Button, centerOf, imageResource, Key, keyboard, mouse, screen, sleep} from "@nut-tree/nut-js";
import "@nut-tree/nl-matcher";
import {useSelenium} from "@nut-tree/selenium-bridge";

let driver;

const setup = async () => {
    const options = new Options();

    screen.config.autoHighlight = true;
    keyboard.config.autoDelayMs = 10;
    screen.config.resourceDirectory = '../non-headless';

    let driver;
    if (process.env.RUN_REMOTE) {
        console.log("Running in remote mode");
        driver = await new Builder()
            .forBrowser(Browser.CHROME)
            .setChromeOptions(options)
            .usingServer("http://selenium:4444/wd/hub")
            .build();
    } else {
        console.log("Running locally");
        if (process.env.RUN_HEADLESS) {
            // screen.config.resourceDirectory = '../headless';
            driver = await new Builder()
                .forBrowser(Browser.CHROME)
                .setChromeOptions(options.addArguments("--headless=new"))
                .build();
        } else {
            driver = await new Builder().forBrowser(Browser.CHROME).build();
        }
        await driver.get('https://google.com');
    }
    return driver;
}

try {
    driver = await setup();

    useSelenium({driver});

    await driver.manage().setTimeouts({implicit: 500});

    const acceptButton = await screen.find(imageResource('target.png'));
    await mouse.move(centerOf(acceptButton));
    await mouse.click(Button.LEFT);

    const searchBox = await screen.waitFor(imageResource('search.png'), 5000, 100);
    await mouse.move(centerOf(searchBox));
    await mouse.click(Button.LEFT);
    await keyboard.type("nut.js")
    await keyboard.type(Key.Enter);

    const result = await screen.waitFor(imageResource('google.png'), 7000, 1000);
    await screen.captureRegion('result.png', result);
} catch (e) {
    console.error(e);
} finally {
    await driver.quit();
}

Buy

@nut-tree/selenium-bridge is included in the Solo and Team plans, as well as the OCR and nl-matcher packages.