9/18/2020

sparse tensor to csr_matrix

from scipy.sparse import csr_matrix
import numpy as np


x = val_data.x
dim = len(x)
print(dim)
edge_index = val_data.edge_index
print(edge_index) #sparse tensor
row = edge_index[0].numpy()
col = edge_index[1].numpy()
edge_num = len(row)
data = np.ones( edge_num )
mtx = csr_matrix((data, (row, col)), shape=(dim, dim))
#print( type(mtx.toarray()), mtx.toarray().shape)
print( mtx.toarray(), type(mtx.toarray()), mtx.toarray().shape) 



let's image 

val_data.x is node features ex) 13x1000

val_data.edge_index is sparse edge index stored by torch tensor


now we want to convert it to csr_matrix

The above code is example for this case.


The print out is like this:

tensor([[ 0,  0,  0,  1,  1,  1,  1,  1,  2,  2,  2,  2,  3,  3,  3,  3,  3,  4,
          4,  4,  4,  4,  4,  5,  5,  5,  5,  5,  6,  6,  6,  6,  6,  7,  7,  7,
          8,  8,  8,  9,  9,  9,  9,  9, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11,
         11, 11, 12, 12, 12, 12],
        [ 1,  3, 10,  0,  2,  3, 10, 11,  1,  3, 11, 12,  0,  1,  2, 11, 12,  5,
          6,  8,  9, 11, 12,  4,  6,  7,  8,  9,  4,  5,  7,  9, 10,  5,  6,  8,
          4,  5,  7,  4,  5,  6, 10, 11,  0,  1,  6,  9, 11,  1,  2,  3,  4,  9,
         10, 12,  2,  3,  4, 11]])
[[0. 1. 0. 1. 0. 0. 0. 0. 0. 0. 1. 0. 0.]
 [1. 0. 1. 1. 0. 0. 0. 0. 0. 0. 1. 1. 0.]
 [0. 1. 0. 1. 0. 0. 0. 0. 0. 0. 0. 1. 1.]
 [1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1.]
 [0. 0. 0. 0. 0. 1. 1. 0. 1. 1. 0. 1. 1.]
 [0. 0. 0. 0. 1. 0. 1. 1. 1. 1. 0. 0. 0.]
 [0. 0. 0. 0. 1. 1. 0. 1. 0. 1. 1. 0. 0.]
 [0. 0. 0. 0. 0. 1. 1. 0. 1. 0. 0. 0. 0.]
 [0. 0. 0. 0. 1. 1. 0. 1. 0. 0. 0. 0. 0.]
 [0. 0. 0. 0. 1. 1. 1. 0. 0. 0. 1. 1. 0.]
 [1. 1. 0. 0. 0. 0. 1. 0. 0. 1. 0. 1. 0.]
 [0. 1. 1. 1. 1. 0. 0. 0. 0. 1. 1. 0. 1.]
 [0. 0. 1. 1. 1. 0. 0. 0. 0. 0. 0. 1. 0.]] <class 'numpy.ndarray'> (13, 13)


Thank you
Enjoy Pytorch!


9/07/2020

image augmentation by python

pip install imgaug
pip install imagecorruptions

github : https://github.com/aleju/imgaug


import numpy as np
import imgaug as ia
import imgaug.augmenters as iaa
import cv2

def agument_rewrite(file_list):

sometimes = lambda aug: iaa.Sometimes(0.1, aug)
seq = iaa.Sequential(
[
# apply the following augmenters to most images
sometimes(iaa.CropAndPad(percent=(-0.02, 0.02), pad_mode=ia.ALL, pad_cval=(0, 255))),
sometimes(iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5)), # add gaussian noise to images
sometimes(iaa.Dropout(p=(0, 0.2))),
sometimes(iaa.CoarseDropout(0.02, size_percent=0.15, per_channel=0.5)),
sometimes(iaa.Solarize(0.5, threshold=(32, 128))),
sometimes(iaa.Cartoon()),
sometimes(iaa.MotionBlur(k=15)),
sometimes(iaa.AllChannelsCLAHE()),
sometimes(iaa.Emboss(alpha=(0.0, 1.0), strength=(0.5, 1.5))),
sometimes(iaa.ElasticTransformation(alpha=(0, 5.0), sigma=0.25)),
sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))),
sometimes(iaa.imgcorruptlike.Snow(severity=2)),
sometimes(iaa.Superpixels(p_replace=0.3, n_segments=500)),
sometimes(iaa.Rain(speed=(0.1, 0.3))),
sometimes(iaa.Snowflakes(flake_size=(0.1, 0.4), speed=(0.01, 0.05))),
sometimes(iaa.Fog()),
],
random_order=True
)

for i, v in enumerate(file_list):
img = cv2.imread(v)
images_aug = seq(images=[img])[0] # done by the library
cv2.imwrite(v, images_aug)
print('{}/{} aug : {}'.format(i, len(file_list), v))

9/06/2020

Fix indention in VS code

 

  • On Windows Shift + Alt + F
  • On Mac Shift + Option + F
  • On Linux Ctrl + Shift + I


8/20/2020

8/18/2020

How to fix Python SSL CERTIFICATE_VERIFY_FAILED

 put this code on the top of code line:


import os, ssl
if (not os.environ.get('PYTHONHTTPSVERIFY', '') and
getattr(ssl, '_create_unverified_context', None)):
ssl._create_default_https_context = ssl._create_unverified_context

Get list from dir and separate train and test (python function)


from sklearn.model_selection import train_test_split
import random
import os
import glob

def train_test_split_from_dir(origin_dir, test_size=0.2):
os.chdir(origin_dir)
#get list
data_list = []
for file in glob.glob("*.jpg"):
data_list.append(file)
random.shuffle(data_list)
train_json_list, test_json_list = train_test_split(data_list, test_size=test_size)

return train_json_list, test_json_list