8/08/2019

PIL to string, string to PIL (python)

It's simple example source code for that:

PIL to string(base64)
- PIL open image
- image to byte
- byte to string (base64)

string(base64) to PIL
- string to byte
- PIL load byte

--
import base64
import io
from PIL import Image

#open file using PIL
pil_img = Image.open('IMG_0510.jpg')
width, height = pil_img.size
print(width, height)


#get image data as byte
buffer = io.BytesIO()
pil_img.save(buffer, format=pil_img.format)
buffer_value = buffer.getvalue()

#byte to string
base64_str = base64.b64encode(buffer_value)

#read string to image buffer
buffer2 = base64.b64decode(base64_str)
pil_img2 = Image.open(io.BytesIO(buffer2))
width2, height2 = pil_img2.size
print(width2, height2)


#check first & second image
pil_img.show()
pil_img2.show()
--

here, another source code for :
OpenCV -> PIL -> resize -> OpenCV

http://study.marearts.com/2019/06/opencv-pil-resize-opencv.html

7/30/2019

Simple example for CNN + MNIST + Keras, Tensorboard, save model, load model

Training Code CNN + MNIST

..

import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Input
from keras.layers import Conv2D, MaxPooling2D

"""Build CNN Model"""
num_classes = 10 input_shape = (28, 28, 1) #mnist channels first format model = Sequential() model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) model.summary() """Download MNIST Data""" from keras.datasets import mnist import numpy as np # the data, split between train and test sets (x_train, y_train), (x_test, y_test) = mnist.load_data() #(60000, 28, 28) -> (60000, 28, 28, 1) x_train = x_train.astype('float32') / 255. x_test = x_test.astype('float32') / 255. x_train = np.reshape(x_train, (len(x_train), 28, 28, 1)) # adapt this if using `channels_first` image data format x_test = np.reshape(x_test, (len(x_test), 28, 28, 1)) # adapt this if using `channels_first` image data format # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) """Show some images""" import matplotlib.pyplot as plt row = 10 col = 10 n = row * col plt.figure(figsize=(4, 4)) for i in range(n): # display original #https://jakevdp.github.io/PythonDataScienceHandbook/04.08-multiple-subplots.html ax = plt.subplot(row, col, i+1) plt.imshow(x_test[i].reshape(28, 28)) plt.gray() ax.get_xaxis().set_visible(False) ax.get_yaxis().set_visible(False) plt.show() """set up tensorboard""" from datetime import datetime import os logdir="logs/scalars/" + datetime.now().strftime("%Y%m%d-%H%M%S") os.makedirs(logdir, exist_ok=True) tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir) """Train model""" from keras.callbacks import TensorBoard batch_size = 128 epochs = 1 model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, shuffle=True, validation_data=(x_test, y_test), callbacks=[TensorBoard(log_dir=logdir)]) score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1]) """test one image data """ x_test[0].shape one_image = x_test[0].reshape(1,28,28,1) y_pred_all = model.predict(one_image) y_pred_it = model.predict_classes(one_image) print(y_pred_all, y_pred_it) plt.imshow(x_test[0].reshape(28, 28)) plt.show() """save model to drive""" model.save('my_cnn_mnist_model.h5')


..
CNN network Layout


Dataset


Run Tensorboard

>cd ./logs/scalars/20190730-105257
>tensorboard --logdir=./


almost 99% accuracy



Load Model and test one mnist image
...
"""load model from drive"""
from keras.models import load_model
new_model = load_model('my_cnn_mnist_model.h5')
"""load 1 image from drive"""
from PIL import Image
import numpy as np
"""test prediction"""
img_path = './mnist_7_450.jpg'
img = Image.open(img_path) #.convert("L") img = np.resize(img, (28,28,1)) im2arr = np.array(img) im2arr = im2arr.reshape(1,28,28,1) y_pred = new_model.predict_classes(im2arr) print(y_pred)

...

Test image


output
[7]


download minist jpeg file on here: http://study.marearts.com/2015/09/mnist-image-data-jpg-files.html



7/01/2019

Check if string matches with regular expression pattern in python

simple code for checking string matched with certain pattern.

import re
file3 = 'keyvalue_reference_1.json'
pattern = re.compile("keyvalue_reference_[0-9]+.json")
re = pattern.match(file3)

if re:
    print('matched')
else:
    print('non matched')


Thank you.

AWS S3, Get object list in Subfolder by python code using s3_client.list_objects function

This is my s3 folder structure





This is code to get file list in certain subfolder.

#get boto3 instance
s3_client = boto3.client(
        's3',
        aws_access_key_id=ACCESS_KEY,
        aws_secret_access_key=SECRET_KEY,
    )

#get object list
contents = s3_client.list_objects(Bucket='test-can-delete-anyone', Prefix='folder1/subfolder1')['Contents']
for object in contents:
     print(object['Key'])


result
folder1/subfolder1/
folder1/subfolder1/4_kitchen.jpg
folder1/subfolder1/5_bathroom.jpg
folder1/subfolder1/5_bedroom.jpg
folder1/subfolder1/5_frontal.jpg
folder1/subfolder1/5_kitchen.jpg
folder1/subfolder1/6_bathroom.jpg

another example
#get object list
contents = s3_client.list_objects(Bucket='test-can-delete-anyone', Prefix='folder1/')['Contents']
for object in contents:
        print(object['Key'])

result
folder1/
folder1/1_kitchen.jpg
folder1/2_bathroom.jpg
folder1/2_bedroom.jpg
folder1/2_frontal.jpg
folder1/2_kitchen.jpg
folder1/subfolder1/
folder1/subfolder1/4_kitchen.jpg
folder1/subfolder1/5_bathroom.jpg
folder1/subfolder1/5_bedroom.jpg
folder1/subfolder1/5_frontal.jpg
folder1/subfolder1/5_kitchen.jpg
folder1/subfolder1/6_bathroom.jpg


AWS s3 bucket - check folder exist or not by python code

Check certain folder exist in s3 bucket by python ncode


-
#create boto3 instance
s3_client = boto3.client(
        's3',
        aws_access_key_id=ACCESS_KEY,
        aws_secret_access_key=SECRET_KEY,
    )

#check folder exist
try:
        s3_client.get_object(Bucket='s3-bucket-name', Key='folder-name/')
        print('folder exist')
except botocore.exceptions.ClientError as e:
        print('no folder exist')
-

Thank you.


function type is here:
def check_folder_exist(s3_client, bucket_name, folder_name):
    
    try:
        s3_client.get_object(Bucket=bucket_name, Key=folder_name)
        return True
    except botocore.exceptions.ClientError as e:
        return False

6/30/2019

AWS S3 bucket, folder creation in python code

Basically s3 bucket doesn't have folder concept.
But this code create folder by key, and it doesn't have any object.


--
#create boto3 instance
s3_client = boto3.client(
        's3',
        aws_access_key_id=ACCESS_KEY,
        aws_secret_access_key=SECRET_KEY,
    )

#create folder by key    
s3_client.put_object(Bucket='s3-bucket-name', Key=('folder-name'+'/'))
--

Thank you.

6/10/2019

OpenCV -> PIL -> resize -> OpenCV

Some simple code for this processing

1. Read image by OpenCV
2. Conver from OpenCV to PIL image
3. some processing using PIL, ex)resize
4. Conver from PIL to OpenCV

Check this code.


import cv2
from PIL import Image
import numpy

#target resize
r_x = 100
r_y = 100


#read image using opencv
cv_img_o = cv2.imread('A.png')

#conver mat to pil
cv_img = cv2.cvtColor(cv_img_o, cv2.COLOR_BGR2RGB)
im_pil = Image.fromarray(cv_img)

#resize pil
im_pil = im_pil.resize((r_x,r_y), Image.ANTIALIAS)

#convert pil to mat
cv_img_r = numpy.array(im_pil)

# Convert RGB to BGR
cv_img_r = cv2.cvtColor(cv_img_r, cv2.COLOR_RGB2BGR)
#cv_img_r = cv_img_r[:, :, ::-1].copy()

cv2.namedWindow('origin',0)
cv2.imshow('origin', cv_img_o)

cv2.namedWindow('resize',0)
cv2.imshow('resize', cv_img_r)

cv2.waitKey(0)