Template Matching using OpenCV Python

image_gray: np.ndarray
TM_METHOD=cv2.TM_CCOEFF_NORMED

def __init__(self):
    self.take_screenshot()

def take_screenshot(self):
    user32 = ctypes.windll.user32
    user32.SetProcessDPIAware()
    image = ImageGrab.grab()
    # save on file
    # image.save('screenshot.jpg', format='JPEG', quality=20)

    # load image from memory
    image_rgb = np.array(image)

    # load from file
    # image_rgb=cv2.imread('screenshot.jpg')
    self.image_gray = cv2.cvtColor(image_rgb, cv2.COLOR_BGR2GRAY)

@staticmethod
def save_screenshot():
    user32 = ctypes.windll.user32
    user32.SetProcessDPIAware()
    image = ImageGrab.grab()
    image.save("screenshot.jpg")

def template_available(self, template_path: str, threshold=0.9):

    # self.take_screenshot()

    template = cv2.imread(template_path, 0)
    # w, h = template.shape[::-1]

    res = cv2.matchTemplate(self.image_gray, template, self.TM_METHOD)
    loc = np.where(res >= threshold)

    if len(loc[0]) == 0 & len(loc[1]) == 0:
        return False
    else:
        print("template_available : " + template_path)
        return True

def template_count(self, template_path: str, threshold=0.9):

    # self.take_screenshot()

    template = cv2.imread(template_path, 0)
    # w, h = template.shape[::-1]

    res = cv2.matchTemplate(self.image_gray, template, self.TM_METHOD)
    loc = np.where(res >= threshold)

    return len(loc[0])

def template_location(self, template_path: str, threshold=0.9):

    self.take_screenshot()

    template = cv2.imread(template_path, 0)
    w, h = template.shape[::-1]

    res = cv2.matchTemplate(self.image_gray, template, self.TM_METHOD)
    loc = np.where(res >= threshold)

    if len(loc[0]) == 0 & len(loc[1]) == 0:
        return None
    else:
        click_y = loc[0][0] + h / 2
        click_x = loc[1][0] + w / 2

        return click_x, click_y

def move_mouse_to_template(self, template_path: str):
    loc = self.template_location(template_path)

    if loc is not None:
        click_x = loc[0]
        click_y = loc[1]
        pyautogui.moveTo(click_x, click_y)

def click_template(self, template_path: str, threshold=0.9):
    loc = self.template_location(template_path, threshold)

    if loc is not None:
        click_x = loc[0]
        click_y = loc[1]
        pyautogui.click(click_x, click_y)
        pyautogui.FAILSAFE = False
        pyautogui.moveTo(0, 0)
        print("click_template : " + template_path)
        return True

    return False

@staticmethod
def scroll_mouse(value=10):
    pyautogui.scroll(value)
    pyautogui.FAILSAFE = False
    pyautogui.moveTo(0, 0)

@staticmethod
def get_template_path(template: str):
    template_name = f"template{template}.jpg"
    template_path = os.path.join(os.getcwd(), 'templates', template_name)
    return template_path

 

Take Screenshot using Python PIL

# Importing Image and ImageGrab module from PIL package 
from PIL import Image, ImageGrab 
  
# creating an image object 
im1 = Image.open(r"C:\Users\sadow984\Desktop\download2.JPG") 
  
# using the grab method 
im2 = ImageGrab.grab(bbox = None) 
  
im2.show() 

Taking screenshots of specific size or taking screenshots from specific region

# Importing Image and ImageGrab module from PIL package 
from PIL import Image, ImageGrab 
  
# creating an image object 
im1 = Image.open(r"C:\Users\sadow984\Desktop\download2.JPG") 
  
# using the grab method 
im2 = ImageGrab.grab(bbox =(0, 0, 300, 300)) 
  
im2.show() 

Parameters: box – The crop rectangle, as a (left, upper, right, lower)-tuple.

References
https://www.geeksforgeeks.org/pyhton-pil-imagegrab-grab-method/
https://stackoverflow.com/questions/19697210/taking-screen-shots-of-specific-size
https://stackoverflow.com/questions/47337811/what-do-the-parameters-of-bbox-mean-in-pillow

numpy.where() in Python

returns the indices of elements in an input array where the given condition is satisfied.

# Python program explaining 
# where() function 

import numpy as np 

np.where([[True, False], [True, True]], 
    [[1, 2], [3, 4]], [[5, 6], [7, 8]]) 
# Python program explaining 
# where() function 

import numpy as np 

# a is an array of integers. 
a = np.array([[1, 2, 3], [4, 5, 6]]) 

print(a) 

print ('Indices of elements <4') 

b = np.where(a<4) 
print(b) 

print("Elements which are <4") 
print(a[b]) 

References
https://www.geeksforgeeks.org/numpy-where-in-python/

Querying data with Pandas .query() method

Single condition filtering

# importing pandas package 
import pandas as pd 

# making data frame from csv file 
data = pd.read_csv("employees.csv") 

# replacing blank spaces with '_' 
data.columns =[column.replace(" ", "_") for column in data.columns] 

# filtering with query method 
data.query('Senior_Management == True', inplace = True) 

# display 
data 

Multiple condition filtering

# importing pandas package 
import pandas as pd 

# making data frame from csv file 
data = pd.read_csv("employees.csv") 

# replacing blank spaces with '_' 
data.columns =[column.replace(" ", "_") for column in data.columns] 

# filtering with query method 
data.query('Senior_Management == True
      and Gender =="Male" and Team =="Marketing"
      and First_Name =="Johnny"', inplace = True) 

# display 
data 

References
https://www.geeksforgeeks.org/python-filtering-data-with-pandas-query-method/

Template Matching On Images C# using OpenCV

Image<Bgr, byte> Image1 = new Image<Bgr, byte>(Properties.Resources.Image1); //Your first image
        Image<Bgr, byte> Image2 = new Image<Bgr, byte>(Properties.Resources.Image2); //Your second image

        double Threshold = 0.8; //set it to a decimal value between 0 and 1.00, 1.00 meaning that the images must be identical

        Image<Gray, float> Matches = Image1.MatchTemplate(Image2, TemplateMatchingType.CcoeffNormed);

        for (int y = 0; y < Matches.Data.GetLength(0); y++)
        {
            for (int x = 0; x < Matches.Data.GetLength(1); x++)
            {
                 if (Matches.Data[y, x, 0] >= Threshold) //Check if its a valid match
                 {
                     //Image2 found within Image1
                 }
            }
        }

References
https://stackoverflow.com/questions/39570033/template-matching-on-images-c-sharp

Move and Click Mouse in C#

[DllImport("user32.dll")]
public static extern void mouse_event(MouseEventFlags dwFlags, uint dx, uint dy, uint dwData, MouseEventDataXButtons dwExtraInfo);

[DllImport("User32.Dll")]
public static extern long SetCursorPos(int x, int y);
[Flags]
public enum MouseEventFlags : uint
{
    LEFTDOWN = 0x00000002,
    LEFTUP = 0x00000004,
    MIDDLEDOWN = 0x00000020,
    MIDDLEUP = 0x00000040,
    MOVE = 0x00000001,
    ABSOLUTE = 0x00008000,
    RIGHTDOWN = 0x00000008,
    RIGHTUP = 0x00000010,
    WHEEL = 0x00000800,
    XDOWN = 0x00000080,
    XUP = 0x00000100
}

//Use the values of this enum for the 'dwData' parameter
//to specify an X button when using MouseEventFlags.XDOWN or
//MouseEventFlags.XUP for the dwFlags parameter.
public enum MouseEventDataXButtons : uint
{
    XBUTTON1 = 0x00000001,
    XBUTTON2 = 0x00000002
}
public static void MoveMouse(int x, int y)
{
    PInvoke.SetCursorPos(x, y);
}

public static void ClickMouse(int x, int y)
{
    PInvoke.mouse_event(MouseEventFlags.LEFTDOWN | MouseEventFlags.ABSOLUTE, (uint)x, (uint)y, 0, 0);
    Thread.Sleep(10);
    PInvoke.mouse_event(MouseEventFlags.LEFTUP | MouseEventFlags.ABSOLUTE, (uint)x, (uint)y, 0, 0);
}

References
https://stackoverflow.com/questions/13520705/move-mouse-to-position-and-left-click
http://www.pinvoke.net/default.aspx/user32.mouse_event
https://stackoverflow.com/questions/8050825/how-to-move-mouse-cursor-using-c
https://www.pinvoke.net/default.aspx/user32.setcursorpos