backend/photo_log/autocrop/autocrop.py

import cv2
import imutils
import random
import numpy as np
import math
from PIL import Image, ImageDraw
import itertools
import pytesseract


def preprocess_image(image, blur_params=[20,20,25], canny_params=[25,250]):
	"""
	Turns the image into grayscale, applies a bilateral filter
	to smooth out unimportant parts of the image, and returns
	the Canny filtered result.
	image: cv2.imread image
	"""
	gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
	gray = cv2.bilateralFilter(gray, blur_params[0], blur_params[1], blur_params[2],)
	edged = cv2.Canny(gray, canny_params[0], canny_params[1])
	return edged


def hough_transform(edged):
	"""
	Returns and image representing the hough space of the edge image.
	edged: Canny filtered cv2 image
	"""
	max_d = np.sqrt(edged.shape[0]**2 + edged.shape[1]**2)
	hough_space = [[0] * (2 * math.ceil(max_d)) for i in range(157*2)]
	for y in range(edged.shape[0]):
		for x in range(edged.shape[1]):
			pixel = edged[y][x]
			if not pixel > 0:
				continue
			for alpha in range(157*2):
				d = x * math.cos(alpha/2.0/100.0) + y * math.sin(alpha/2.0/100.0)
				hough_space[alpha][round(d)+math.ceil(max_d)] += 1
	return hough_space


#TODO fix removal on "the other site" of the hough space => negative indexes in try / except
def get_max_params(hough_space, num_params, rm_radius=7):
	"""
	Iterates over the maxima of hough space image.
	After each maximum found a circle of the radius rm_radius is beeing "cut out"
	of the hough space image.
	Returns an array of tuples containing the maximum parameters (alpha, d).
	hough_space: hough space image
	num_params: number of the maxima to be found
	rm_radius: optional = 7; the radius to be cut out around the maximum found 
	"""
	all_max_params = []
	for i in range(num_params):
		hough_array = np.array(hough_space)
		max_params = np.unravel_index(hough_array.argmax(), hough_array.shape)

		if -math.inf in max_params:
			break

		alpha = max_params[0]/2.0/100.0
		d = max_params[1] - hough_array.shape[1]/2.0
		all_max_params.append((alpha, d))

		for yi in range(rm_radius*2+1):
				for xi in range(rm_radius*2+1):
					if math.sqrt((-rm_radius+yi)**2 + (-rm_radius+xi)**2) <= rm_radius:
						try:
							hough_space[abs(max_params[0]-rm_radius+yi)][abs(max_params[1]-rm_radius+xi)] = -math.inf
						except Exception:
							pass

	return all_max_params, hough_space


def _draw_hough_space(hough_space):
	max_val = np.amax(hough_space)
	img = Image.new(mode="RGB", size=(len(hough_space[0]), len(hough_space)))
	pixels = img.load()
	for y in range(len(hough_space)):
		for x in range(len(hough_space[0])):
			if hough_space[y][x] == -math.inf:
				hough_space[y][x] = 0
			val = int(hough_space[y][x] / float(max_val) * 255)
			pixels[x,y] = (val,val,val)
	return img


def _draw_lines(image, all_max_params, resize_height):
	img = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
	img = Image.fromarray(img)
	new_height = resize_height
	new_width  = int(new_height * img.size[0] / img.size[1])
	img = img.resize((new_width, new_height), Image.ANTIALIAS)
	pixels = img.load()

	for (alpha,d) in all_max_params:
		dist = 0.5 # line thickness (distance image pixels to line coordinates)
		for y in range(img.size[1]):
			for x in range(img.size[0]):
				val = (x * math.cos(alpha) + y * math.sin(alpha)) - d
				if val <= dist and val >= -dist:
					pixels[x,y] = (0,200,255)
	
	return img


def get_intersect(a1, a2, b1, b2):
    """ 
    Returns the point of intersection of the lines passing through a2,a1 and b2,b1.
    a1: [x, y] a point on the first line
    a2: [x, y] another point on the first line
    b1: [x, y] a point on the second line
    b2: [x, y] another point on the second line
    """
    s = np.vstack([a1,a2,b1,b2])        # s for stacked
    h = np.hstack((s, np.ones((4, 1)))) # h for homogeneous
    l1 = np.cross(h[0], h[1])           # get first line
    l2 = np.cross(h[2], h[3])           # get second line
    x, y, z = np.cross(l1, l2)          # point of intersection
    if z == 0:                          # lines are parallel
        return (float('inf'), float('inf'))
    return (x/z, y/z)


def insert_intersection(intersections, params1, params2, inters_pt):
	if not params1 in intersections:
		intersections[params1] = []
	intersections[params1].append((params2, inters_pt))

	if not params2 in intersections:
		intersections[params2] = []
	intersections[params2].append((params1, inters_pt))


# TODO fix wrong random intersection points
def find_intersections(all_max_params, img_size, allowed_angle_diff=3):
	"""
	Takes hough space parameters of found lines and returns a list of tuples of
	intersection points.
	all_max_params: (alpha, d) tuple list of hough space parameters
	img_size: the size of the in which the intersections are to be found in
	"""
	intersections_by_lines = {}
	all_intersections = []

	other_max_params = all_max_params.copy()
	for (alpha1,d1) in all_max_params:
		for (alpha2,d2) in other_max_params:
			if alpha1 == alpha2 and d1 == d2:
				continue
			
			y = random.randint(0, img_size[0])
			x = (d1 - (y * math.sin(alpha1))) / math.cos(alpha1)
			a1 = [x,y]

			y = random.randint(0, img_size[0])
			x = (d1 - (y * math.sin(alpha1))) / math.cos(alpha1)
			a2 = [x,y]

			y = random.randint(0, img_size[0])
			x = (d2 - (y * math.sin(alpha2))) / math.cos(alpha2)
			b1 = [x,y]

			y = random.randint(0, img_size[0])
			x = (d2 - (y * math.sin(alpha2))) / math.cos(alpha2)
			b2 = [x,y]

			# get intersection point of two lines, where each line
			# is given by two random points of each line
			# a1, a1: two [x,y] points on line a
			# b1, b1: two [x,y] points on line b
			inters = get_intersect(a1,a2, b1,b2)
			
			# are lines parallel or is intersection outside of the image?
			if math.inf in inters:
				continue
			if inters[0] < 0 or inters[0] >= img_size[1]:
				continue
			if inters[1] < 0 or inters[1] >= img_size[0]:
				continue

			# calculate vectors of each line 1 and 2
			p1 = [a1[0]-a2[0], a1[1]-a2[1]]
			p2 = [b1[0]-b2[0], b1[1]-b2[1]]

			inters = (round(inters[0]),round(inters[1]))

			try:
				dot_product = p1[0]*p2[0] + p1[1]*p2[1]
				abs_p1 = math.sqrt(p1[0]**2 + p1[1]**2)
				abs_p2 = math.sqrt(p2[0]**2 + p2[1]**2)

				angle = math.degrees(math.acos(dot_product / (abs_p1 * abs_p2)))
				angle_diff = abs(abs(angle) - 90)
			except ValueError as e:
				print(e)
				continue

			if not(angle_diff > allowed_angle_diff):
				all_intersections.append(inters)
				params1 = (alpha1, d1)
				params2 = (alpha2, d2)
				insert_intersection(intersections_by_lines, params1, params2, inters)

		other_max_params.remove((alpha1, d1))

	return intersections_by_lines, all_intersections


def _draw_intersections(image, intersections):
	image = image.copy()
	for inters in intersections:
		color = (0,255,0)
		pt_radius = 1
		draw = ImageDraw.Draw(image)
		draw.ellipse((inters[0]-pt_radius,inters[1]-pt_radius,inters[0]+pt_radius,inters[1]+pt_radius), fill=color, outline=color)
	return image


def get_n_connections(start, intersections_by_lines, n):
	if n <= 0:
		return [[start]]
	
	connections = []
	for line in intersections_by_lines[start]:
		neighbors_connections = get_n_connections(line[0], intersections_by_lines, n-1)
		for con in neighbors_connections:
			if not start in con:
				if len(con) == 1:
					connections.append([start, con[0]])
				else:
					connections.append([start] + con)

	return connections


def get_cycles(intersections_by_lines, n=4):
	start = list(intersections_by_lines.keys())[0]
	connections = get_n_connections(start, intersections_by_lines, n=n-1)

	cycles = []
	for connection in connections:
		last_vertex = connection[-1]
		# can we get back to the beginning?
		if start in [con[0] for con in intersections_by_lines[last_vertex]]:
			cycles.append(connection + [start])
	
	return cycles


def get_intersection_points_from_lines(rects_lines, intersections_by_lines):
	rects_points = []
	for rect in rects_lines:
		points = []
		for i in range(len(rect)-1):
			line1 = rect[i]
			line2 = rect[i+1]
			points.append(list(filter(lambda con: con[0]==line2, intersections_by_lines[line1]))[0][1])
		points.append(points[0])
		rects_points.append(points)
	return rects_points


def remove_small_rects(rects_points, image_shape, min_image_coverage=0.3):
	image_area = image_shape[0] * image_shape[1]
	possible_rects = []
	for rect in rects_points:
		pt1 = rect[0]
		pt2 = rect[1]
		pt3 = rect[2]
		len_side1 = math.sqrt((pt1[0]-pt2[0])**2 + (pt1[1]-pt2[1])**2)
		len_side2 = math.sqrt((pt2[0]-pt3[0])**2 + (pt2[1]-pt3[1])**2)

		area = len_side1 * len_side2

		if area >= min_image_coverage * image_area:
			possible_rects.append(rect)
	return possible_rects


def _draw_rects(rects_points, image):
	image_lines = image.copy()
	draw = ImageDraw.Draw(image_lines)
	for rect in rects_points:
		rect = [(pt[0],pt[1]) for pt in rect]
		draw.line(rect, width=2, fill='yellow')
	return image_lines


def crop_warp_image(image, rect):
	#rect = np.asarray(rect[:4])
	
	rect = np.array(rect,dtype = "float32").reshape(4,2)

	ordered_points = np.zeros((4, 2), dtype = "float32")
	s = rect.sum(axis = 1)
	ordered_points[0] = rect[np.argmin(s)]
	ordered_points[2] = rect[np.argmax(s)]

	diff = np.diff(rect, axis = 1)
	ordered_points[1] = rect[np.argmin(diff)]
	ordered_points[3] = rect[np.argmax(diff)]

	(tl, tr, br, bl) = ordered_points
	widthA = np.sqrt(((br[0] - bl[0]) ** 2) + ((br[1] - bl[1]) ** 2))
	widthB = np.sqrt(((tr[0] - tl[0]) ** 2) + ((tr[1] - tl[1]) ** 2))
	heightA = np.sqrt(((tr[0] - br[0]) ** 2) + ((tr[1] - br[1]) ** 2))
	heightB = np.sqrt(((tl[0] - bl[0]) ** 2) + ((tl[1] - bl[1]) ** 2))

	maxWidth = max(int(widthA), int(widthB))
	maxHeight = max(int(heightA), int(heightB))

	dst_rect = np.array([
		[0, 0],
		[maxWidth - 1, 0],
		[maxWidth - 1, maxHeight - 1],
		[0, maxHeight - 1]], dtype = "float32")

	M = cv2.getPerspectiveTransform(ordered_points, dst_rect)
	warped_img = cv2.warpPerspective(image, M, (maxWidth, maxHeight))

	return warped_img


def get_rect_bounding_box(rect):
	min_x = min([pt[0] for pt in rect])
	max_x = max([pt[0] for pt in rect])
	min_y = min([pt[1] for pt in rect])
	max_y = max([pt[1] for pt in rect])

	bbox = [[min_x, min_y], [max_x, min_y], [max_x, max_y], [min_x, max_y]]

	return bbox


def _cv2ImageToPIL(cv2_image):
	#cv2_image = cv2.cvtColor(cv2_image, cv2.COLOR_BGR2RGB)
	pil_image = Image.fromarray(cv2_image)
	return pil_image


def _PILImageToCv2(pil_image):
	cv2_image = np.asarray(pil_image)
	return cv2_image


def autocrop(original_image, DEBUG=False):
	"""
	Automatically crops an image to size of the flip chart in the given image,
	and returns a cropped PIL image.
	image: the PIL image to crop
	"""

	resize_height = 300
	blur_params = [35,15,40] # size, color, space
	canny_params = [80,250]
	num_hough_params = 50
	hough_rm_radius = 20
	line_angle_diff = 3
	min_image_coverage = 0.3

	original_image = _PILImageToCv2(original_image)

	# shrink image to a smaller size to speed up autocropping
	ratio = original_image.shape[0] / float(resize_height)
	image_small = imutils.resize(original_image, height=resize_height)

	edge_image = preprocess_image(image_small, blur_params=blur_params, canny_params=canny_params)
	
	if DEBUG:
		cv2.imwrite('debug_out/edges.png', edge_image)
	
	hough_space = hough_transform(edge_image)
	all_max_params, hough_space = get_max_params(hough_space, num_params=num_hough_params, rm_radius=hough_rm_radius)

	if DEBUG:
		hough_space_image = _draw_hough_space(hough_space)
		hough_space_image.save("debug_out/hough.png")
		image_lines = _draw_lines(original_image, all_max_params, resize_height)

	intersections_by_lines, all_intersections = find_intersections(all_max_params, image_small.shape, allowed_angle_diff=line_angle_diff)

	if DEBUG:
		image_lines = _draw_intersections(image_lines, all_intersections)

	rects_lines = get_cycles(intersections_by_lines)
	rects_points = get_intersection_points_from_lines(rects_lines, intersections_by_lines)
	rects_points = remove_small_rects(rects_points, image_small.shape, min_image_coverage=min_image_coverage)

	if DEBUG:
		image_lines = _draw_rects(rects_points, image_lines)
		image_lines.save("debug_out/line.png")

	# TODO: find best rectangle
	try:
		best_rect = rects_points[0]
		best_rect = [(round(pt[0]*ratio), round(pt[1]*ratio)) for pt in best_rect]
		best_rect = best_rect[:4]

		cropped_image = crop_warp_image(original_image, best_rect)

		cropped_image = _cv2ImageToPIL(cropped_image)

		bbox = get_rect_bounding_box(best_rect)
	except Exception as e:
		cropped_image = _cv2ImageToPIL(original_image)
		best_rect = None
		bbox = None

	intersections_as_list = [[round(i[0]*ratio), round(i[1]*ratio)] for i in all_intersections]

	return cropped_image, best_rect, bbox, intersections_as_list


if __name__ == "__main__":
	#img = Image.open('example_imgs/bill.png')
	img = Image.open('example_imgs/image4.jpg')

	cropped_image, rect, bbox, intersections = autocrop(img, DEBUG=True)

	cropped_image.save('debug_out/cropped.png')
	print(rect)
	print(bbox)
	print(intersections)