人工智能与谎言和欺骗

import numpy as np import matplotlib.pyplot as plt from matplotlib.colors import NoNorm %matplotlib inline import math from tqdm import tqdm #from joblib import Parallel, delayed from skimage.draw import ellipse, polygon from keras import Model from keras.optimizers import Adam from keras.layers import Input,Conv2D,Conv2DTranspose,MaxPooling2D,concatenate from keras.layers import BatchNormalization,Activation,Add,Dropout from keras.losses import binary_crossentropy from keras import backend as K from keras.models import load_model import tensorflow as tf import keras as keras w_size = 128 train_num = 10000 radius_min = 10 radius_max = 30 

 def dice_coef(y_true, y_pred): y_true_f = K.flatten(y_true) y_pred = K.cast(y_pred, 'float32') y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32') intersection = y_true_f * y_pred_f score = 2. * K.sum(intersection) / (K.sum(y_true_f) + K.sum(y_pred_f)) return score def dice_loss(y_true, y_pred): smooth = 1. y_true_f = K.flatten(y_true) y_pred_f = K.flatten(y_pred) intersection = y_true_f * y_pred_f score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth) return 1. - score def bce_dice_loss(y_true, y_pred): return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred) def get_iou_vector(A, B): # Numpy version batch_size = A.shape[0] metric = 0.0 for batch in range(batch_size): t, p = A[batch], B[batch] true = np.sum(t) pred = np.sum(p) # deal with empty mask first if true == 0: metric += (pred == 0) continue # non empty mask case. Union is never empty # hence it is safe to divide by its number of pixels intersection = np.sum(t * p) union = true + pred - intersection iou = intersection / union # iou metrric is a stepwise approximation of the real iou over 0.5 iou = np.floor(max(0, (iou - 0.45)*20)) / 10 metric += iou # teake the average over all images in batch metric /= batch_size return metric def my_iou_metric(label, pred): # Tensorflow version return tf.py_func(get_iou_vector, [label, pred > 0.5], tf.float64) from keras.utils.generic_utils import get_custom_objects get_custom_objects().update({'bce_dice_loss': bce_dice_loss }) get_custom_objects().update({'dice_loss': dice_loss }) get_custom_objects().update({'dice_coef': dice_coef }) get_custom_objects().update({'my_iou_metric': my_iou_metric }) 

 def build_model(input_layer, start_neurons): # 128 -> 64 conv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(input_layer) conv1 = Conv2D(start_neurons * 1, (3, 3), activation="relu", padding="same")(conv1) pool1 = Conv2D(start_neurons * 1, (2, 2), strides=(2, 2), activation="relu", padding="same")(conv1) # pool1 = Dropout(0.25)(pool1) # 64 -> 32 conv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(pool1) conv2 = Conv2D(start_neurons * 2, (3, 3), activation="relu", padding="same")(conv2) pool2 = Conv2D(start_neurons * 1, (2, 2), strides=(2, 2), activation="relu", padding="same")(conv2) # pool2 = Dropout(0.5)(pool2) # 32 -> 16 conv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(pool2) conv3 = Conv2D(start_neurons * 4, (3, 3), activation="relu", padding="same")(conv3) pool3 = Conv2D(start_neurons * 1, (2, 2), strides=(2, 2), activation="relu", padding="same")(conv3) # pool3 = Dropout(0.5)(pool3) # 16 -> 8 conv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(pool3) conv4 = Conv2D(start_neurons * 8, (3, 3), activation="relu", padding="same")(conv4) pool4 = Conv2D(start_neurons * 1, (2, 2), strides=(2, 2), activation="relu", padding="same")(conv4) # pool4 = Dropout(0.5)(pool4) # Middle convm = Conv2D(start_neurons * 16, (3, 3), activation="relu", padding="same")(pool4) convm = Conv2D(start_neurons * 16, (3, 3) , activation="relu", padding="same")(convm) # 8 -> 16 deconv4 = Conv2DTranspose(start_neurons * 8, (3, 3), strides=(2, 2), padding="same")(convm) uconv4 = concatenate([deconv4, conv4]) # uconv4 = Dropout(0.5)(uconv4) uconv4 = Conv2D(start_neurons * 8, (3, 3) , activation="relu", padding="same")(uconv4) uconv4 = Conv2D(start_neurons * 8, (3, 3) , activation="relu", padding="same")(uconv4) # 16 -> 32 deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="same")(uconv4) uconv3 = concatenate([deconv3, conv3]) # uconv3 = Dropout(0.5)(uconv3) uconv3 = Conv2D(start_neurons * 4, (3, 3) , activation="relu", padding="same")(uconv3) uconv3 = Conv2D(start_neurons * 4, (3, 3) , activation="relu", padding="same")(uconv3) # 32 -> 64 deconv2 = Conv2DTranspose(start_neurons * 2, (3, 3), strides=(2, 2), padding="same")(uconv3) uconv2 = concatenate([deconv2, conv2]) # uconv2 = Dropout(0.5)(uconv2) uconv2 = Conv2D(start_neurons * 2, (3, 3) , activation="relu", padding="same")(uconv2) uconv2 = Conv2D(start_neurons * 2, (3, 3) , activation="relu", padding="same")(uconv2) # 64 -> 128 deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="same")(uconv2) uconv1 = concatenate([deconv1, conv1]) # uconv1 = Dropout(0.5)(uconv1) uconv1 = Conv2D(start_neurons * 1, (3, 3) , activation="relu", padding="same")(uconv1) uconv1 = Conv2D(start_neurons * 1, (3, 3) , activation="relu", padding="same")(uconv1) # uncov1 = Dropout(0.5)(uconv1) output_layer = Conv2D(1, (1,1), padding="same", activation="sigmoid")(uconv1) return output_layer input_layer = Input((w_size, w_size, 1)) output_layer = build_model(input_layer, 27) model = Model(input_layer, output_layer) model.compile(loss=bce_dice_loss, optimizer=Adam(lr=1e-4), metrics=[my_iou_metric]) model.summary() 

 def make_sh(f_imgs, f_msks, val_len): precision = 0.85 batch_size = 50 t = tqdm() t_batch_size = 50 raw_len = val_len id_train = 1 #id_select = 1 v_false = np.zeros((train_num), dtype='float') while True: if id_train == 1: fit = model.fit(f_imgs[m2_select>0], f_msks[m2_select>0], batch_size=batch_size, epochs=1, verbose=0 ) current_accu = fit.history['my_iou_metric'][0] current_loss = fit.history['loss'][0] if current_accu > precision: id_train = 0 else: t_pred = model.predict( f_imgs[raw_len: min(raw_len+t_batch_size,f_imgs.shape[0])], batch_size=batch_size ) for kk in range(t_pred.shape[0]): val_iou = get_iou_vector( f_msks[raw_len+kk].reshape(1,w_size,w_size,1), t_pred[kk].reshape(1,w_size,w_size,1) > 0.5) v_false[raw_len+kk] = val_iou if val_iou < precision*0.95: new_img_test = 1 m2_select[raw_len+kk] = 1 val_len += 1 break raw_len += (kk+1) id_train = 1 t.set_description("Accuracy {0:6.4f} loss {1:6.4f} selected img {2:5d} tested img {3:5d} ". format(current_accu, current_loss, val_len, raw_len)) t.update(1) if raw_len >= train_num: break t.close() return v_false