Day 84 – Marker Pen Painting Under 5 Mins Using Stylized Neural Painting Artificial Intelligence - IT Consultant - SAP, Artificial Intelligence and Machine Learning

This is the third series on Stylized Neural Painting, as we have seen below posts.

Oil Painting Under 5 Mins Using Stylized Neural Painting Artificial Intelligence
Watercolor Painting Under 5 Mins Using Stylized Neural Painting Artificial Intelligence

In this blog, we do marker pen painting and see how it comes.

Let’s copy the code from Github Stylized Neural Painting and run it in my local Google Colab with minor tweaks.

Citation

@inproceedings{zou2020stylized,
title={Stylized Neural Painting},
author={Zhengxia Zou and Tianyang Shi and Shuang Qiu and Yi Yuan and Zhenwei Shi},
year={2020},
eprint={2011.08114},
archivePrefix={arXiv},
primaryClass={cs.CV}
}

Copy to Clipboard

# settings
parser = argparse.ArgumentParser(description='STYLIZED NEURAL PAINTING')
args = parser.parse_args(args=[])
args.img_path = 'jar_image.jpg' # path to input photo
args.renderer = 'markerpen' # [watercolor, markerpen, oilpaintbrush, rectangle]
args.canvas_color = 'black' # [black, white]
args.canvas_size = 512 # size of the canvas for stroke rendering'
args.keep_aspect_ratio = False # whether to keep input aspect ratio when saving outputs
args.max_m_strokes = 500 # max number of strokes
args.max_divide = 5 # divide an image up-to max_divide x max_divide patches
args.beta_L1 = 1.0 # weight for L1 loss
args.with_ot_loss = False # set True for imporving the convergence by using optimal transportation loss, but will slow-down the speed
args.beta_ot = 0.1 # weight for optimal transportation loss
args.net_G = 'zou-fusion-net' # renderer architecture
args.renderer_checkpoint_dir = './checkpoints_G_markerpen' # dir to load the pretrained neu-renderer
args.lr = 0.005 # learning rate for stroke searching
args.output_dir = './output' # dir to save painting results
args.disable_preview = True # disable cv2.imshow, for running remotely without x-display

Copy to Clipboard

# Define some helper functions for downloading pretrained model
# taken from this StackOverflow answer: https://stackoverflow.com/a/39225039
import requests

def download_file_from_google_drive(id, destination):
    URL = "https://docs.google.com/uc?export=download"

session = requests.Session()

response = session.get(URL, params = { 'id' : id }, stream = True)
    token = get_confirm_token(response)

if token:
        params = { 'id' : id, 'confirm' : token }
        response = session.get(URL, params = params, stream = True)

save_response_content(response, destination)

def get_confirm_token(response):
    for key, value in response.cookies.items():
        if key.startswith('download_warning'):
            return value

return None

def save_response_content(response, destination):
    CHUNK_SIZE = 32768

with open(destination, "wb") as f:
        for chunk in response.iter_content(CHUNK_SIZE):
            if chunk: # filter out keep-alive new chunks
                f.write(chunk)

Copy to Clipboard

def optimize_x(pt):

pt._load_checkpoint()
    pt.net_G.eval()

print('begin drawing...')

PARAMS = np.zeros([1, 0, pt.rderr.d], np.float32)

if pt.rderr.canvas_color == 'white':
        CANVAS_tmp = torch.ones([1, 3, 128, 128]).to(device)
    else:
        CANVAS_tmp = torch.zeros([1, 3, 128, 128]).to(device)

for pt.m_grid in range(1, pt.max_divide + 1):

pt.img_batch = utils.img2patches(pt.img_, pt.m_grid, pt.net_G.out_size).to(device)
        pt.G_final_pred_canvas = CANVAS_tmp

pt.initialize_params()
        pt.x_ctt.requires_grad = True
        pt.x_color.requires_grad = True
        pt.x_alpha.requires_grad = True
        utils.set_requires_grad(pt.net_G, False)

pt.optimizer_x = optim.RMSprop([pt.x_ctt, pt.x_color, pt.x_alpha], lr=pt.lr, centered=True)

pt.step_id = 0
        for pt.anchor_id in range(0, pt.m_strokes_per_block):
            pt.stroke_sampler(pt.anchor_id)
            iters_per_stroke = int(500 / pt.m_strokes_per_block)
            for i in range(iters_per_stroke):
                pt.G_pred_canvas = CANVAS_tmp

# update x
                pt.optimizer_x.zero_grad()

pt.x_ctt.data = torch.clamp(pt.x_ctt.data, 0.1, 1 - 0.1)
                pt.x_color.data = torch.clamp(pt.x_color.data, 0, 1)
                pt.x_alpha.data = torch.clamp(pt.x_alpha.data, 0, 1)

pt._forward_pass()
                pt._drawing_step_states()
                pt._backward_x()

pt.x_ctt.data = torch.clamp(pt.x_ctt.data, 0.1, 1 - 0.1)
                pt.x_color.data = torch.clamp(pt.x_color.data, 0, 1)
                pt.x_alpha.data = torch.clamp(pt.x_alpha.data, 0, 1)

pt.optimizer_x.step()
                pt.step_id += 1

v = pt._normalize_strokes(pt.x)
        v = pt._shuffle_strokes_and_reshape(v)
        PARAMS = np.concatenate([PARAMS, v], axis=1)
        CANVAS_tmp = pt._render(PARAMS, save_jpgs=False, save_video=False)
        CANVAS_tmp = utils.img2patches(CANVAS_tmp, pt.m_grid + 1, pt.net_G.out_size).to(device)

pt._save_stroke_params(PARAMS)
    final_rendered_image = pt._render(PARAMS, save_jpgs=False, save_video=True)

return final_rendered_image

Copy to Clipboard

iteration step 484, G_loss: 0.12134, step_acc: 15.22069, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 485, G_loss: 0.12132, step_acc: 15.22380, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 486, G_loss: 0.12128, step_acc: 15.22147, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 487, G_loss: 0.12133, step_acc: 15.21938, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 488, G_loss: 0.12137, step_acc: 15.21919, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 489, G_loss: 0.12136, step_acc: 15.22158, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 490, G_loss: 0.12134, step_acc: 15.22599, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 491, G_loss: 0.12128, step_acc: 15.22406, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 492, G_loss: 0.12130, step_acc: 15.22164, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 493, G_loss: 0.12134, step_acc: 15.22901, grid_scale: 5 / 5, strokes: 9 / 9
iteration step 494, G_loss: 0.12123, step_acc: 15.22797, grid_scale: 5 / 5, strokes: 9 / 9
rendering canvas...
saving stroke parameters...
rendering canvas...

Copy to Clipboard

The output of the marker pen painting generated by Stylized Neural Painting as below.