DEV Community

Mostafa Gazar
Mostafa Gazar

Posted on

TensorFlow 2.x and Keras must keep snippets

#tensorflow #keras #machinelearning #beginners

MNIST classification using Sequential API 

# Source: https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/quickstart/beginner.ipynb

import tensorflow as tf
mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()
# Normalize the inputs
x_train, x_test = x_train / 255., x_test / 255.

model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),
  tf.keras.layers.Dense(128, activation='relu'),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10, activation='softmax')
])

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

model.fit(x_train, y_train, epochs=5)
model.evaluate(x_test, y_test)
Enter fullscreen mode Exit fullscreen mode

MNIST classification using Functional API 

# Source: https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/quickstart/advanced.ipynb

import tensorflow as tf
mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()
# Normalize the inputs
x_train, x_test = x_train / 255., x_test / 255.

class MyModel(tf.keras.Model):
  def __init__(self):
    super(MyModel, self).__init__()
    self.conv1 = Conv2D(32, 3, activation='relu')
    self.flatten = Flatten()
    self.d1 = Dense(128, activation='relu')
    self.d2 = Dense(10, activation='softmax')

  def call(self, x):
    x = self.conv1(x)
    x = self.flatten(x)
    x = self.d1(x)
    return self.d2(x)
model = MyModel()

loss_object = tf.keras.losses.SparseCategoricalCrossentropy()
optimizer = tf.keras.optimizers.Adam()

with tf.GradientTape() as tape:
  logits = model(images)
  loss_value = loss_object(logits, labels)
grads = tape.gradient(loss_value, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
Enter fullscreen mode Exit fullscreen mode

Eager execution

TF 2.0 runs in eager execution mode by default

Eager execution is great for debugging becuse it evaluates operations immediately, without building graphs. Operations return concrete values instead of constructing a computational graph to run later.

Use @tf.function annotation to transform a subset of Python syntax into portable, high-performance TensorFlow graphs.

@tf.function
def increment_even(x):
  if x % 2 == 0:
    x += 1

  return x
Enter fullscreen mode Exit fullscreen mode

Use ImageDataGenerator to load raw images in batches with real-time data augmentation 

from tensorflow.keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(
    # Normalize the input
    rescale=1/255,
    # Augmentation specs
    rotation_range=40,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True
)
test_datagen = ImageDataGenerator(
    # Normalize the input
    rescale=1/255
)

train_generator = train_datagen.flow_from_directory(
    some_path,  # Path to your data.
    target_size=(150, 150),
    batch_size=batch_size,  # Your training batch size
    class_mode='categorical'  # You can also `binary` for binary classication tasks.
)

test_generator = test_datagen.flow_from_directory(
    # Use the same parameters as in train_generator.
)

# Finally we train the model
model.fit_generator(
    train_generator,
    steps_per_epoch=len(train_generator),
    epochs=50  # You can also add validation generators here.
)

# Evaluate the model
loss, accuracy = model.evaluate_generator(
    test_generator,
    steps=len(test_generator)
)
Enter fullscreen mode Exit fullscreen mode

Keras data preprocessing utils API

Text classification using Sequential API 

import tensorflow as tf
from tensorflow.keras import datasets
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.preprocessing.sequence import pad_sequences

# Load data
vocab_size = 10000
# If you are loading your own corpus you will need to tokenize it first using `tensorflow.keras.preprocessing.text.Tokenizer.fit_on_texts`
(train_x, train_y), (test_x, test_y) = datasets.reuters.load_data(num_words=vocab_size)

# Pad the sequences so that they would have the same length
max_length = 200
train_x = pad_sequences(train_x, maxlen=max_length)
test_x = pad_sequences(test_x, maxlen=max_length)

# One hot encode y
number_of_classes = len(np.unique(train_y))
train_y = to_categorical(train_y, number_of_classes)
test_y = to_categorical(test_y, number_of_classes)

# Create the model
model = Sequential()
model.add(layers.Embedding(vocab_size, 1024, input_length=max_length))
model.add(layers.GlobalAveragePooling1D())
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(46, activation='softmax'))
model.summary()

# Training...
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
model.fit(x=train_x, y=train_y, validation_split=0.2, shuffle=True, batch_size=512, epochs=20)

# Evaluate the model 
loss, accuracy = model.evaluate(x=test_x, y=test_y)
Enter fullscreen mode Exit fullscreen mode

Tokenize corpus using Tokenizer.fit_on_texts 

from tensorflow.keras.preprocessing.text import Tokenizer

# Create the tokenizer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(corpus)

# Summary
print(tokenizer.word_counts)
print(tokenizer.document_count)
print(tokenizer.word_index)
print(tokenizer.word_docs)
Enter fullscreen mode Exit fullscreen mode

Load pre-trained embeddings 

from pathlib import Path

vocab_size = 10000
embedding_size = 100

# Build your model
model = Sequential()
embedding_layer = layers.Embedding(vocab_size, embedding_size, input_length=max_length)
model.add(embedding_layer)
# Add more layers...

# Download pre-trained word embeddings
# I will use the smallest available pre-trained word vectors from [GloVe](https://nlp.stanford.edu/projects/glove/) 
# which should be more than enough. That was glove.6B.zip which consists of 6B tokens, 400K vocab, uncased, 50d, 100d, 200d, & 300d vectors. 
# It is an 822 MB download.
embedding_path = Path("../embedding")
embedding_path.mkdir(parents=True, exist_ok=True)

!wget -O {embedding_path}/glove.6B.zip http://nlp.stanford.edu/data/glove.6B.zip
!unzip {embedding_path}/glove.6B.zip -d {embedding_path}

# Load embeddings
embedding_index = {}
with open(embedding_path/"glove.6B.100d.txt", "r") as reader:
    line = reader.readline()
    while line != '':  # The EOF char is an empty string
        values = line.split()
        word = values[0]
        coefs = np.asarray(values[1:], np.float32)
        embedding_index[word] = coefs

        line = reader.readline()

# Set embedding_layer weights
embedding_matrix = np.zeros((vocab_size, embedding_size))
for word, index in word_index.items():
    if index >= vocab_size:
        continue

    coefs = embedding_index.get(word)
    if coefs is not None:
        embedding_matrix[index] = coefs

embedding_layer.set_weights([embedding_matrix])
embedding_layer.trainable = False

# TODO :: Compile model here for trainable change to take effect
# model.compile()
Enter fullscreen mode Exit fullscreen mode

Plot training accuracy and loss after training is done 

import matplotlib.pyplot as plt
%matplotlib inline

# Train model.
history = model.fit(x=train_x, y=train_y, validation_split=0.2, shuffle=True, batch_size=128, epochs=10)
print(history.history.keys())

# Plot training progress.
fig = plt.figure(figsize=(20, 10))

plt.subplot(1, 2, 1)
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('Model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')

plt.subplot(1, 2, 2)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')

plt.show()
Enter fullscreen mode Exit fullscreen mode

Track training progress in TensorBoard 

from tensorflow import keras

callbacks = [
    keras.callbacks.TensorBoard(log_dir='../logs/dogs_vs_cats_v2')
]

history = model.fit_generator(
    ...
    callbacks=callbacks,
    ...
)

# Then check progress in TensorBoard by running `tensorboard --logdir=logs/dogs_vs_cats_v2` in terminal
Enter fullscreen mode Exit fullscreen mode

Dynamic learning rate (learning rate schedules) 

from tensorflow import keras

initial_learning_rate = 0.1
learning_rate_schedule = keras.optimizers.schedules.ExponentialDecay(
    initial_learning_rate,
    decay_steps=100000,
    decay_rate=0.96,
    staircase=True
)

optimizer = keras.optimizers.RMSprop(learning_rate=learning_rate_schedule)
Enter fullscreen mode Exit fullscreen mode

Another option would be to us ReduceLROnPlateau callback. ReduceLROnPlateau reduces learning rate when a metric stop improving.

from tensorflow import keras

reduce_lr = keras.callbacks.ReduceLROnPlateau(
    monitor='val_loss', 
    factor=0.1,  # Divide the learning rate by 10 when triggered
    patience=5,  # Trigger when val_loss has stopped improving for 5 epochs 
    min_lr=0.001
)

model.fit(X_train, Y_train, callbacks=[reduce_lr])
Enter fullscreen mode Exit fullscreen mode

Handy callbacks 

from tensorflow import keras

callbacks = [
    # Stop the training when there is no improvement in the validation loss for three consecutive epochs.
    keras.callbacks.EarlyStopping(
        monitor='val_loss', 
        patience=3
    ),
    # Save model after every epoch or save the best one so far only 
    keras.callbacks.ModelCheckpoint(
        file_path="_model.h5",
        monitor="val_loss",
        save_best_only=True
    ),
    # Check training progress in TensorBoard
    keras.callbacks.TensorBoard(
        log_dir="model_log_dir",
        histogram_freq=1,  # Records activation histograms every 1 epoch
        embedding_freq = 1  # Record emedding data every 1 epoch. If set to 0, embeddings won't be visualized.
    )
]

model.fit(X_train, Y_train, callbacks=callbacks)
Enter fullscreen mode Exit fullscreen mode

Save model for inference 

model.save('path_to_my_model.h5')

# Load the model for inference
model = keras.models.load_model('path_to_my_model.h5')

model.trainable = False
# TODO :: Compile model here for trainable change to take effect
# model.compile()

# Make sure this model does not have trainable variables
model.trainable_variables
Enter fullscreen mode Exit fullscreen mode

Save model checkpoints and continue training where you left off 

# Source: https://www.tensorflow.org/alpha/guide/checkpoints
import tensorflow as tf

opt = tf.keras.optimizers.Adam(0.1)
net = Net()

ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=opt, net=net)
manager = tf.train.CheckpointManager(ckpt, 'tf_ckpts', max_to_keep=3)
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
    print(f'Restored from {manager.latest_checkpoint}')
else:
    print('Initializing from scratch.')

for example in toy_dataset():
    loss = train_step(net, example, opt)
    ckpt.step.assign_add(1)
    if int(ckpt.step) % 10 == 0:
        save_path = manager.save()
        print(f'Saved checkpoint for step {int(ckpt.step)}: {save_path}')
        print(f'loss {loss.numpy():1.2f}')
Enter fullscreen mode Exit fullscreen mode

I am working on a project called ML Studio, want to get early access to and product updates? Subscribe here or follow me on twitter.

Top comments (0)

Read next

abdibrokhim profile image

I access over 200 AI models with one API. Here’s how…

Ibrohim Abdivokhidov -

oliverbennet profile image

I Started Learning Docker a Year Ago, Now I make my Own Containers.

Oliver Bennet -

mikeyoung44 profile image

TokenFormer: Rethinking Transformer Scaling with Tokenized Model Parameters

Mike Young -

mikeyoung44 profile image

New Research Breaks Through AI Language Model Safeguards, Exposing Security Risks

Mike Young -