Examples

Color Search

Find and interact with UI elements based on pixel colors

colorpixelstatus-detection

Color search allows you to find pixels or detect UI states based on color. This is useful for status indicators, validation states, and visual feedback detection.

Reading Pixel Colors

typescript
import { screen, Point } from "@nut-tree/nut-js";

async function getPixelColor(x: number, y: number) {
  const color = await screen.colorAt(new Point(x, y));

  console.log(`Pixel at (${x}, ${y}):`);
  console.log(`  Red: ${color.R}`);
  console.log(`  Green: ${color.G}`);
  console.log(`  Blue: ${color.B}`);
  console.log(`  Alpha: ${color.A}`);

  return color;
}

Detect Status Indicator

Check the color of a status light to determine state:

typescript
import { screen, Point } from "@nut-tree/nut-js";

type Status = "success" | "error" | "warning" | "unknown";

async function getStatusIndicator(x: number, y: number): Promise<Status> {
  const color = await screen.colorAt(new Point(x, y));

  // Green = success (high G, low R and B)
  if (color.G > 200 && color.R < 100 && color.B < 100) {
    return "success";
  }

  // Red = error (high R, low G and B)
  if (color.R > 200 && color.G < 100 && color.B < 100) {
    return "error";
  }

  // Yellow/Orange = warning (high R and G, low B)
  if (color.R > 200 && color.G > 150 && color.B < 100) {
    return "warning";
  }

  return "unknown";
}

// Usage
const status = await getStatusIndicator(50, 100);
console.log(`Current status: ${status}`);

if (status === "error") {
  console.error("Error detected! Taking screenshot...");
  await screen.capture("error-state.png");
}

Wait for Color Change

Poll until a pixel changes to an expected color:

typescript
import { screen, Point } from "@nut-tree/nut-js";

async function waitForGreen(
  x: number,
  y: number,
  timeoutMs: number = 10000
): Promise<boolean> {
  const startTime = Date.now();

  while (Date.now() - startTime < timeoutMs) {
    const color = await screen.colorAt(new Point(x, y));

    if (color.G > 200 && color.R < 100 && color.B < 100) {
      console.log("Indicator turned green!");
      return true;
    }

    // Wait 100ms before checking again
    await new Promise(r => setTimeout(r, 100));
  }

  throw new Error("Timed out waiting for green indicator");
}

// Wait for build status to turn green
await waitForGreen(50, 100, 30000);

Generic Color Wait Function

typescript
import { screen, Point, RGBA } from "@nut-tree/nut-js";

interface ColorCondition {
  R?: { min?: number; max?: number };
  G?: { min?: number; max?: number };
  B?: { min?: number; max?: number };
}

function matchesCondition(color: RGBA, condition: ColorCondition): boolean {
  if (condition.R) {
    if (condition.R.min !== undefined && color.R < condition.R.min) return false;
    if (condition.R.max !== undefined && color.R > condition.R.max) return false;
  }
  if (condition.G) {
    if (condition.G.min !== undefined && color.G < condition.G.min) return false;
    if (condition.G.max !== undefined && color.G > condition.G.max) return false;
  }
  if (condition.B) {
    if (condition.B.min !== undefined && color.B < condition.B.min) return false;
    if (condition.B.max !== undefined && color.B > condition.B.max) return false;
  }
  return true;
}

async function waitForColor(
  x: number,
  y: number,
  condition: ColorCondition,
  timeoutMs: number = 10000
): Promise<RGBA> {
  const startTime = Date.now();

  while (Date.now() - startTime < timeoutMs) {
    const color = await screen.colorAt(new Point(x, y));

    if (matchesCondition(color, condition)) {
      return color;
    }

    await new Promise(r => setTimeout(r, 100));
  }

  throw new Error("Timed out waiting for color condition");
}

// Usage: wait for any green-ish color
await waitForColor(50, 100, {
  G: { min: 150 },
  R: { max: 100 },
  B: { max: 100 },
});

Sample Multiple Points

Read colors from multiple positions for gradient verification or multi-indicator checks:

typescript
import { screen, Point, RGBA } from "@nut-tree/nut-js";

interface ColorSample {
  x: number;
  y: number;
  color: RGBA;
}

async function sampleGradient(
  startX: number,
  y: number,
  count: number,
  spacing: number
): Promise<ColorSample[]> {
  const samples: ColorSample[] = [];

  for (let i = 0; i < count; i++) {
    const x = startX + (i * spacing);
    const color = await screen.colorAt(new Point(x, y));
    samples.push({ x, y, color });
  }

  return samples;
}

// Sample 10 points across a gradient
const gradient = await sampleGradient(100, 200, 10, 20);
gradient.forEach(({ x, color }) => {
  console.log(`x=${x}: RGB(${color.R}, ${color.G}, ${color.B})`);
});

Check Multiple Status Indicators

typescript
import { screen, Point } from "@nut-tree/nut-js";

interface StatusIndicator {
  name: string;
  x: number;
  y: number;
}

type StatusResult = "success" | "error" | "warning" | "unknown";

async function checkAllIndicators(
  indicators: StatusIndicator[]
): Promise<Map<string, StatusResult>> {
  const results = new Map<string, StatusResult>();

  for (const indicator of indicators) {
    const color = await screen.colorAt(new Point(indicator.x, indicator.y));

    let status: StatusResult = "unknown";
    if (color.G > 200 && color.R < 100) {
      status = "success";
    } else if (color.R > 200 && color.G < 100) {
      status = "error";
    } else if (color.R > 200 && color.G > 150) {
      status = "warning";
    }

    results.set(indicator.name, status);
  }

  return results;
}

// Usage
const indicators: StatusIndicator[] = [
  { name: "Database", x: 50, y: 100 },
  { name: "API", x: 50, y: 130 },
  { name: "Cache", x: 50, y: 160 },
];

const statuses = await checkAllIndicators(indicators);
statuses.forEach((status, name) => {
  console.log(`${name}: ${status}`);
});

Finding Pixels by Color

typescript
import { screen, RGBA } from "@nut-tree/nut-js";
import { pixelWithColor } from "@nut-tree/nut-js";

async function findRedPixel() {
  const targetColor: RGBA = { R: 255, G: 0, B: 0, A: 255 };

  // Find first pixel of that color
  const location = await screen.find(pixelWithColor(targetColor));
  console.log(`Found red pixel at: ${location.left}, ${location.top}`);

  return location;
}

async function findAllRedPixels() {
  const targetColor: RGBA = { R: 255, G: 0, B: 0, A: 255 };

  // Find all pixels of that color
  const allRed = await screen.findAll(pixelWithColor(targetColor));
  console.log(`Found ${allRed.length} red pixels`);

  return allRed;
}

Color Comparison with Tolerance

typescript
import { screen, Point, RGBA } from "@nut-tree/nut-js";

function colorsMatch(c1: RGBA, c2: RGBA, tolerance: number = 10): boolean {
  return (
    Math.abs(c1.R - c2.R) <= tolerance &&
    Math.abs(c1.G - c2.G) <= tolerance &&
    Math.abs(c1.B - c2.B) <= tolerance
  );
}

async function findColorWithTolerance(
  targetColor: RGBA,
  tolerance: number = 10
) {
  const width = await screen.width();
  const height = await screen.height();

  // Sample grid of points
  for (let y = 0; y < height; y += 10) {
    for (let x = 0; x < width; x += 10) {
      const color = await screen.colorAt(new Point(x, y));
      if (colorsMatch(color, targetColor, tolerance)) {
        return { x, y, color };
      }
    }
  }

  return null;
}

Detect Theme (Light/Dark Mode)

typescript
import { screen, Point, Region } from "@nut-tree/nut-js";

async function detectTheme(): Promise<"light" | "dark"> {
  // Sample the background color from a known location
  const bgColor = await screen.colorAt(new Point(100, 100));

  // Calculate luminance
  const luminance = (0.299 * bgColor.R + 0.587 * bgColor.G + 0.114 * bgColor.B);

  return luminance > 128 ? "light" : "dark";
}

async function getThemeAwareCoordinates() {
  const theme = await detectTheme();

  // Return different coordinates based on theme
  // (useful when UI elements shift between themes)
  if (theme === "dark") {
    return { buttonX: 150, buttonY: 200 };
  } else {
    return { buttonX: 155, buttonY: 205 };
  }
}

Progress Bar Detection

typescript
import { screen, Point, RGBA } from "@nut-tree/nut-js";

async function getProgressBarPercentage(
  startX: number,
  endX: number,
  y: number,
  filledColor: RGBA,
  tolerance: number = 20
): Promise<number> {
  const totalWidth = endX - startX;
  let filledWidth = 0;

  for (let x = startX; x <= endX; x++) {
    const color = await screen.colorAt(new Point(x, y));

    const matches = (
      Math.abs(color.R - filledColor.R) <= tolerance &&
      Math.abs(color.G - filledColor.G) <= tolerance &&
      Math.abs(color.B - filledColor.B) <= tolerance
    );

    if (matches) {
      filledWidth = x - startX;
    } else {
      break; // Progress bar is contiguous
    }
  }

  return Math.round((filledWidth / totalWidth) * 100);
}

// Usage: check a green progress bar
const progress = await getProgressBarPercentage(
  100, 500, 300,
  { R: 76, G: 175, B: 80, A: 255 } // Material green
);
console.log(`Progress: ${progress}%`);

Best Practices

Use Regions for Performance

typescript
import { screen, Point, Region } from "@nut-tree/nut-js";

// Don't search the entire screen
// Instead, limit to where you expect the element
const statusBarRegion = new Region(0, 0, 200, 50);

async function checkStatusInRegion() {
  // Sample a few points within the known region
  const color = await screen.colorAt(new Point(
    statusBarRegion.left + 25,
    statusBarRegion.top + 25
  ));

  return color;
}

Account for Anti-Aliasing

typescript
import { screen, Point, RGBA } from "@nut-tree/nut-js";

// Sample multiple nearby pixels and average them
async function getAverageColor(centerX: number, centerY: number): Promise<RGBA> {
  const samples: RGBA[] = [];

  for (let dx = -1; dx <= 1; dx++) {
    for (let dy = -1; dy <= 1; dy++) {
      const color = await screen.colorAt(new Point(centerX + dx, centerY + dy));
      samples.push(color);
    }
  }

  return {
    R: Math.round(samples.reduce((sum, c) => sum + c.R, 0) / samples.length),
    G: Math.round(samples.reduce((sum, c) => sum + c.G, 0) / samples.length),
    B: Math.round(samples.reduce((sum, c) => sum + c.B, 0) / samples.length),
    A: 255
  };
}

Was this page helpful?