Erase certain colors with OpenCV + PySimpleGUI
Introduction
In order to make effective use of OpenCV, it is necessary to produce the desired result not only by one process but also by multiple processes and combination with other packages. This time, as an example, let's perform the process of erasing the color of the clicked part.
What you can do with this article
Left-clicking on a particular color in the video will remove that color. To adjust the size of the erased part and the degree of blurring, it is necessary to change parameters such as the HSV value. The parameters can be changed with the GUI.
Process flow
The process is as follows. Loading video
↓
Mask making → Convert the image to HSV and set it to mask only a specific color. To extract the color, left-click on the video to extract the color of that part. In addition, it is possible to make fine adjustments with the slider created with PySimple GUI.
↓
Noise removal to mask image (Opening, Closing) → Since noise remains only by color extraction, noise is removed by Opening and Closing processing.
↓
Swelling process (Dilation) to mask image → Since the outline of the color cannot be extracted well in many cases, the mask part is uniformly expanded.
↓
Repair treatment (Inpaint) on the mask part → Performs the process of repairing the mask part with the surrounding color.
↓
Blur on the mask → Since it stands out only with Inpaint, it is blurred.
↓
display
program
Loading video
import PySimpleGUI as sg
import cv2
import numpy as np
from pathlib import Path
def file_read():
'''
Select a file to read
'''
fp = ""
#GUI layout
layout = [
[
sg.FileBrowse(key="file"),
sg.Text("File"),
sg.InputText()
],
[sg.Submit(key="submit"), sg.Cancel("Exit")]
]
# self.WINDOW generation
window = sg.Window("File selection", layout)
#Event loop
while True:
event, values = window.read(timeout=100)
if event == 'Exit' or event == sg.WIN_CLOSED:
break
elif event == 'submit':
if values[0] == "":
sg.popup("No file has been entered.")
event = ""
else:
fp = values[0]
break
window.close()
return Path(fp)
Color detection by HSV
This function creates a mask to process only the part of the HSV value within the specified range. Specified on PySimpleGUI Reads the value of the slider and generates a mask image.
def hsv(frame, H_max, H_min, S_max, S_min, V_max, V_min, reverse=False):
frame_hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
if reverse:
lower1 = np.array([0, int(S_min), int(V_min)])
upper1 = np.array([int(H_min), int(S_max), int(V_max)])
mask1 = cv2.inRange(frame_hsv, lower1, upper1)
lower2 = np.array([int(H_max), int(S_min), int(V_min)])
upper2 = np.array([255, int(S_max), int(V_max)])
mask2 = cv2.inRange(frame_hsv, lower2, upper2)
mask = mask1 + mask2
frame = cv2.bitwise_and(frame, frame, mask=mask)
# mask = cv2.bitwise_and(frame, mask, mask=mask)
else:
lower = np.array([int(H_min), int(S_min), int(V_min)])
upper = np.array([int(H_max), int(S_max), int(V_max)])
mask = cv2.inRange(frame_hsv, lower, upper)
frame = cv2.bitwise_and(frame, frame, mask=mask)
return frame
Image loading and GUI
Load the video from the file and display it in OpenCV. When you left-click on the image, create a function that reflects the HSV value of that part in the slider value of PySimpleGUI, and specify it in the callback function.
class Main:
def __init__(self):
self.fp = file_read()
self.cap = cv2.VideoCapture(str(self.fp))
#Video save flag
self.rec_flg = False
#Get the first frame
#Check if it can be obtained
self.ret, self.f_frame = self.cap.read()
self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
#If you can get the frame, get various parameters
if self.ret:
self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
#Acquisition of video information
self.fps = self.cap.get(cv2.CAP_PROP_FPS)
self.org_width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH))
self.width = self.org_width
self.org_height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
self.height = self.org_height
self.total_count = self.cap.get(cv2.CAP_PROP_FRAME_COUNT)
#Definition of mask image
self.mask = np.zeros_like(self.f_frame[:, :, 0])
#Frame related
self.frame_count = 0
self.s_frame = 0
self.e_frame = self.total_count
#Playback pause flag
self.stop_flg = False
#GUI event
self.event = ""
cv2.namedWindow("Movie")
#Mouse event callback registration
cv2.setMouseCallback("Movie", self.onMouse)
#Exit if the frame could not be obtained
else:
sg.Popup("Failed to read the file.")
return
#Mouse event
def onMouse(self, event, x, y, flags, param):
if event == cv2.EVENT_LBUTTONUP:
hsv = cv2.cvtColor(
self.frame[y:y + 1, x:x + 1, :], cv2.COLOR_BGR2HSV)
h = int(hsv[:, :, 0])
h_min = max(h - 20, 0)
h_max = min(255, h + 20)
s = int(hsv[:, :, 1])
s_min = max(s - 20, 0)
s_max = min(255, s + 20)
v = int(hsv[:, :, 2])
v_min = max(v - 20, 0)
v_max = min(255, v + 20)
self.window['-H_MIN SLIDER_MASK-'].update(h_min)
self.window['-H_MAX SLIDER_MASK-'].update(h_max)
self.window['-S_MIN SLIDER_MASK-'].update(s_min)
self.window['-S_MAX SLIDER_MASK-'].update(s_max)
self.window['-V_MIN SLIDER_MASK-'].update(v_min)
self.window['-V_MAX SLIDER_MASK-'].update(v_max)
self.window['-Hue Reverse_MASK-'].update(False)
def run(self):
# GUI #######################################################
#GUI layout
layout = [
[
sg.Text("Start", size=(8, 1)),
sg.Slider(
(0, self.total_count - 1),
0,
1,
orientation='h',
size=(45, 15),
key='-START FRAME SLIDER-',
enable_events=True
)
],
[sg.Slider(
(0, self.total_count - 1),
0,
1,
orientation='h',
size=(50, 15),
key='-PROGRESS SLIDER-',
enable_events=True
)],
[sg.HorizontalSeparator()],
[
sg.Text("Resize ", size=(13, 1)),
sg.Slider(
(0.1, 4),
1,
0.01,
orientation='h',
size=(40, 15),
key='-RESIZE SLIDER-',
enable_events=True
)
],
[
sg.Checkbox(
"Inpaint",
size=(10, 1),
default=False,
key='-INPAINT-',
enable_events=True
)
],
[
sg.Checkbox(
"Opening",
size=(20, 1),
default=False,
key='-OPENING-',
enable_events=True
),
sg.Checkbox(
"Closing",
size=(20, 1),
default=False,
key='-CLOSING-',
enable_events=True
),
sg.Slider(
(3, 31),
5,
2,
orientation='h',
size=(15, 15),
key='-OPENING SLIDER-',
enable_events=True
)
],
[
sg.Checkbox(
"Dilation",
size=(10, 1),
default=False,
key='-DILATION-',
enable_events=True
),
sg.Slider(
(1, 31),
4,
2,
orientation='h',
size=(15, 15),
key='-DILATION SLIDER-',
enable_events=True
)
],
[
sg.Checkbox(
'blur',
size=(10, 1),
key='-BLUR-',
enable_events=True
),
sg.Slider(
(1, 10),
1,
1,
orientation='h',
size=(40, 15),
key='-BLUR SLIDER-',
enable_events=True
)
],
[
sg.Text(
'hsv',
size=(10, 1),
key='-HSV_MASK-',
enable_events=True
),
sg.Button('Blue', size=(10, 1)),
sg.Button('Green', size=(10, 1)),
sg.Button('Red', size=(10, 1))
],
[
sg.Checkbox(
'Hue Reverse',
size=(10, 1),
key='-Hue Reverse_MASK-',
enable_events=True
)
],
[
sg.Text('Hue', size=(10, 1), key='-Hue_MASK-'),
sg.Slider(
(0, 255),
0,
1,
orientation='h',
size=(19.4, 15),
key='-H_MIN SLIDER_MASK-',
enable_events=True
),
sg.Slider(
(1, 255),
125,
1,
orientation='h',
size=(19.4, 15),
key='-H_MAX SLIDER_MASK-',
enable_events=True
)
],
[
sg.Text('Saturation', size=(10, 1), key='-Saturation_MASK-'),
sg.Slider(
(0, 255),
50,
1,
orientation='h',
size=(19.4, 15),
key='-S_MIN SLIDER_MASK-',
enable_events=True
),
sg.Slider(
(1, 255),
255,
1,
orientation='h',
size=(19.4, 15),
key='-S_MAX SLIDER_MASK-',
enable_events=True
)
],
[
sg.Text('Value', size=(10, 1), key='-Value_MASK-'),
sg.Slider(
(0, 255),
50,
1,
orientation='h',
size=(19.4, 15),
key='-V_MIN SLIDER_MASK-',
enable_events=True
),
sg.Slider(
(1, 255),
255,
1,
orientation='h',
size=(19.4, 15),
key='-V_MAX SLIDER_MASK-',
enable_events=True
)
],
[sg.Output(size=(65, 5), key='-OUTPUT-')],
[sg.Button('Clear')]
]
# self.Generate Window
self.window = sg.Window('OpenCV Integration', layout, location=(0, 0))
#Display of video information
self.event, values = self.window.read(timeout=0)
print("The file has been read.")
print("File Path: " + str(self.fp))
print("fps: " + str(int(self.fps)))
print("width: " + str(self.width))
print("height: " + str(self.height))
print("frame count: " + str(int(self.total_count)))
#Main loop#########################################################
try:
while True:
#Loading GUI events
self.event, values = self.window.read(
timeout=0
)
#Show event in window
if self.event != "__TIMEOUT__":
print(self.event)
#Exit when the Exit button is pressed or when the window close button is pressed
if self.event in ('Exit', sg.WIN_CLOSED, None):
break
#Reload video
#Works when the start frame is set
if self.event == 'Reset':
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
self.window['-PROGRESS SLIDER-'].update(self.frame_count)
self.video_stabilization_flg = False
self.stab_prepare_flg = False
#Continue to reflect changes to Progress slider
continue
#Frame operation################################################
#Priority is given if the slider is changed directly
if self.event == '-PROGRESS SLIDER-':
#Set the frame count to the progress bar
self.frame_count = int(values['-PROGRESS SLIDER-'])
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
#If you change the start frame
if self.event == '-START FRAME SLIDER-':
self.s_frame = int(values['-START FRAME SLIDER-'])
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
self.window['-PROGRESS SLIDER-'].update(self.frame_count)
#If the counter exceeds the end frame, restart from the start frame
if self.frame_count >= self.e_frame:
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
self.window['-PROGRESS SLIDER-'].update(self.frame_count)
continue
#Pause video loading with the stop button
if self.event == 'Play / Stop':
self.stop_flg = not self.stop_flg
#Unless the stop flag is set and an event occurs, count in
#Stop the operation
#If the stop button is pressed, the video processing will be stopped, but something
#If an event occurs, only update the image
#The same applies when operating the mouse
if(
(
self.stop_flg
and self.event == "__TIMEOUT__"
)
):
self.window['-PROGRESS SLIDER-'].update(self.frame_count)
continue
#Load frame##############################################
self.ret, self.frame = self.cap.read()
#When the last frame is over self.s_Resume from frame
if not self.ret:
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.s_frame)
self.frame_count = self.s_frame
continue
#resize
self.width = int(self.org_width * values['-RESIZE SLIDER-'])
self.height = int(self.org_height * values['-RESIZE SLIDER-'])
self.frame = cv2.resize(self.frame, (self.width, self.height))
#Perform processing for ROI##########################################
if self.event == 'Blue':
self.window['-H_MIN SLIDER_MASK-'].update(70)
self.window['-H_MAX SLIDER_MASK-'].update(110)
self.window['-S_MIN SLIDER_MASK-'].update(70)
self.window['-S_MAX SLIDER_MASK-'].update(255)
self.window['-V_MIN SLIDER_MASK-'].update(0)
self.window['-V_MAX SLIDER_MASK-'].update(255)
self.window['-Hue Reverse_MASK-'].update(False)
if self.event == 'Green':
self.window['-H_MIN SLIDER_MASK-'].update(20)
self.window['-H_MAX SLIDER_MASK-'].update(70)
self.window['-S_MIN SLIDER_MASK-'].update(70)
self.window['-S_MAX SLIDER_MASK-'].update(255)
self.window['-V_MIN SLIDER_MASK-'].update(0)
self.window['-V_MAX SLIDER_MASK-'].update(255)
self.window['-Hue Reverse_MASK-'].update(False)
if self.event == 'Red':
self.window['-H_MIN SLIDER_MASK-'].update(20)
self.window['-H_MAX SLIDER_MASK-'].update(110)
self.window['-S_MIN SLIDER_MASK-'].update(70)
self.window['-S_MAX SLIDER_MASK-'].update(255)
self.window['-V_MIN SLIDER_MASK-'].update(0)
self.window['-V_MAX SLIDER_MASK-'].update(255)
self.window['-Hue Reverse_MASK-'].update(True)
self.mask = self.frame
self.mask = hsv(
self.mask,
values['-H_MAX SLIDER_MASK-'],
values['-H_MIN SLIDER_MASK-'],
values['-S_MAX SLIDER_MASK-'],
values['-S_MIN SLIDER_MASK-'],
values['-V_MAX SLIDER_MASK-'],
values['-V_MIN SLIDER_MASK-'],
values['-Hue Reverse_MASK-']
)
Processing to mask image
Performs grayscale processing, opening processing, Closing processing, and dilation processing of mask images.
#Grayscale
self.mask = cv2.cvtColor(
self.mask,
cv2.COLOR_BGR2GRAY
)
#Noise removal
if values['-OPENING-']:
self.mask = cv2.morphologyEx(self.mask, cv2.MORPH_OPEN,
np.ones((int(values['-OPENING SLIDER-']), int(values['-OPENING SLIDER-'])), np.uint8))
#Noise removal 2
if values['-CLOSING-']:
self.mask = cv2.morphologyEx(self.mask, cv2.MORPH_CLOSE,
np.ones((int(values['-OPENING SLIDER-']), int(values['-OPENING SLIDER-'])), np.uint8))
#Expansion process
if values['-DILATION-']:
self.mask = cv2.dilate(self.mask,
np.ones((int(values['-DILATION SLIDER-']), int(values['-DILATION SLIDER-'])), np.uint8), iterations=1)
Repair processing to the frame, blurring processing
Inpaint processing and Blur processing are performed on the mask part. The part to be inpainted can be implemented by specifying the mask image as an argument. For Blur, we use cv2.bitwise_not to apply Blur only to the mask part.
if values['-INPAINT-']:
self.frame = cv2.inpaint(
self.frame,
self.mask,
2,
cv2.INPAINT_TELEA
)
#Blur
if values['-BLUR-']:
self.frame_roi = cv2.GaussianBlur(
self.frame, (21, 21), values['-BLUR SLIDER-']
)
#Apply mask inside frame
#Mask processing part only.Change to frame
self.frame = cv2.bitwise_not(
cv2.bitwise_not(self.frame_roi),
self.frame,
mask=self.mask
)
#Display of number of frames and elapsed seconds
cv2.putText(self.frame,
str("framecount: {0:.0f}".format(self.frame_count)),
(15,
20),
cv2.FONT_HERSHEY_SIMPLEX,
0.5,
(240,
230,
0),
1,
cv2.LINE_AA)
cv2.putText(self.frame,
str("time: {0:.1f} sec".format(self.frame_count / self.fps)),
(15,
40),
cv2.FONT_HERSHEY_SIMPLEX,
0.5,
(240,
230,
0),
1,
cv2.LINE_AA)
#Display image
cv2.imshow("Movie", self.frame)
cv2.imshow("Mask", cv2.cvtColor(self.mask, cv2.COLOR_GRAY2BGR))
if self.stop_flg:
self.cap.set(cv2.CAP_PROP_POS_FRAMES, self.frame_count)
else:
self.frame_count += 1
self.window['-PROGRESS SLIDER-'].update(
self.frame_count + 1)
#Other processing###############################################
#Clear log window
if self.event == 'Clear':
self.window['-OUTPUT-'].update('')
finally:
cv2.destroyWindow("Movie")
cv2.destroyWindow("Mask")
self.cap.release()
self.window.close()
if __name__ == '__main__':
Main().run()
Reference link
Automatic image interpolation with OpenCV and Python (Fast Marching Method, Navier-Stokes) Image Inpainting
Recommended Posts