Image Matching

import {
  screen,
  mouse,
  straightTo,
  centerOf,
  imageResource,
} from '@nut-tree/nut-js'
import { useNlMatcher } from "@nut-tree/nl-matcher";

useNlMatcher();

async function clickOnImage() {
  // Find an image on screen
  const result = await screen.find(imageResource('button.png'))

  // Move to center and click
  await mouse.move(straightTo(centerOf(result)))
  await mouse.click()
}

import { screen, imageResource } from '@nut-tree/nut-js'
import { useNlMatcher } from "@nut-tree/nl-matcher";

useNlMatcher();

// Set global confidence (0.0 - 1.0)
screen.config.confidence = 0.9

async function findWithConfidence() {
  // Or per-search confidence
  const result = await screen.find(imageResource('icon.png'), {
    confidence: 0.85,
  })

  return result
}

import { screen, imageResource } from '@nut-tree/nut-js'
import { useNlMatcher } from "@nut-tree/nl-matcher";

useNlMatcher();

async function waitForImage() {
  // Wait up to 10 seconds for image to appear (timeout, interval)
  const result = await screen.waitFor(imageResource('loading-complete.png'), 10000, 500)

  console.log('Image appeared at:', result)
}

import { screen, mouse, straightTo, centerOf, imageResource } from '@nut-tree/nut-js'
import { useNlMatcher } from "@nut-tree/nl-matcher";

useNlMatcher();

async function findAllMatches() {
  // Find all instances of an image
  const matches = await screen.findAll(imageResource('list-item.png'))

  console.log(`Found ${matches.length} matches`)

  // Click each one
  for (const match of matches) {
    await mouse.move(straightTo(centerOf(match)))
    await mouse.click()
  }
}

import { screen, Region, imageResource } from '@nut-tree/nut-js'
import { useNlMatcher } from "@nut-tree/nl-matcher";

useNlMatcher();

async function searchInRegion() {
  // Define search region (left, top, width, height)
  const searchArea = new Region(100, 100, 800, 600)

  // Search only in that region
  const result = await screen.find(imageResource('target.png'), {
    searchRegion: searchArea,
  })

  return result
}

import { screen, mouse, straightTo, centerOf, singleWord } from '@nut-tree/nut-js'
import { useOcrPlugin, configure, LanguageModelType, Language } from '@nut-tree/plugin-ocr'

// Configure OCR
useOcrPlugin();
configure({
  dataPath: "./ocr-data",
  languageModelType: LanguageModelType.BEST
});

async function findByText() {
  // Find text on screen using singleWord
  const result = await screen.find(singleWord('Submit'), {
    providerData: {
      lang: [Language.English]
    }
  })

  await mouse.move(straightTo(centerOf(result)))
  await mouse.click()
}

async function findTextCaseInsensitive() {
  // Find text with case-insensitive matching
  const result = await screen.find(singleWord('order'), {
    providerData: {
      lang: [Language.English],
      partialMatch: true,
      caseSensitive: false
    }
  })

  console.log('Found order at:', result)
}

import { screen, Region } from '@nut-tree/nut-js'

async function captureScreenshots() {
  // Full screen capture
  await screen.capture('full-screen.png')

  // Capture specific region
  const region = new Region(0, 0, 400, 300)
  await screen.captureRegion('partial.png', region)

  // Get image data without saving
  const imageData = await screen.grab()
  console.log(`Screen size: ${imageData.width}x${imageData.height}`)
}

import { screen, Region, mouse } from '@nut-tree/nut-js'

async function createReferenceImage() {
  // Position your UI element, then capture it
  console.log('Position the element and press Enter...')

  // Capture a region around the mouse cursor
  const pos = await mouse.getPosition()
  const region = new Region(
    pos.x - 50,
    pos.y - 25,
    100,
    50
  )

  await screen.captureRegion('reference-images/my-button.png', region)
  console.log('Reference image saved!')
}

import { screen, imageResource } from '@nut-tree/nut-js'

// For Retina/HiDPI displays
screen.config.scaleMultiplier = 2

async function findOnHiDPI() {
  // nut.js handles scaling automatically
  const result = await screen.find(imageResource('button@2x.png'))
  return result
}

import { screen, Point, RGBA, pixelWithColor } from '@nut-tree/nut-js'

async function findByColor() {
  // Define the target color
  const red: RGBA = { R: 255, G: 0, B: 0, A: 255 }

  // Find first pixel of that color
  const result = await screen.find(pixelWithColor(red))
  console.log(`Found red pixel at: ${result.left}, ${result.top}`)

  // Find all pixels of that color
  const allRed = await screen.findAll(pixelWithColor(red))
  console.log(`Found ${allRed.length} red pixels`)

  return result
}

async function readColorAtPosition() {
  // Get the color at a specific position
  const color = await screen.colorAt(new Point(100, 200))
  console.log(`Color: R=${color.R}, G=${color.G}, B=${color.B}, A=${color.A}`)
}

import { screen, Region, imageResource } from '@nut-tree/nut-js'
import { useNlMatcher } from "@nut-tree/nl-matcher";

useNlMatcher();

// Cache images for repeated searches
const cachedButton = imageResource('button.png')

async function optimizedSearch() {
  // 1. Use search regions
  const toolbar = new Region(0, 0, 1920, 100)

  // 2. Lower confidence for faster matching (when appropriate)
  screen.config.confidence = 0.8

  // 3. Use cached image resources
  const result = await screen.find(cachedButton, {
    searchRegion: toolbar,
  })

  return result
}

import { screen, imageResource } from '@nut-tree/nut-js'
import { useNlMatcher } from "@nut-tree/nl-matcher";

useNlMatcher();

async function debugImageMatching() {
  try {
    const result = await screen.find(imageResource('target.png'))
    console.log('Found at:', result)
  } catch (error) {
    // Capture current screen for comparison
    await screen.capture('debug-current-screen.png')
    console.log('Image not found. Screen captured for debugging.')

    // Check if the reference image exists
    // Compare with captured screen manually
  }
}

import { screen, imageResource } from '@nut-tree/nut-js'
import { useNlMatcher } from "@nut-tree/nl-matcher";

useNlMatcher();
import path from 'path'

function platformImage(name: string) {
  const platform = process.platform // 'darwin', 'win32', 'linux'
  return imageResource(path.join('images', platform, name))
}

async function crossPlatformClick() {
  const button = await screen.find(platformImage('submit-button.png'))
  // ... click the button
}