# Universal Approximation Theorem - Part 3

## Wrapping Up

You’ve made it to the final part of this blog series! It’s been a fun journey seeing just how flexible neural networks are - so flexible that they have an unlimited ability to overfit. The first two posts in the series were a bit negative and lacked the optimism that is typically associated with any kind of ML. Luckily, today gives us the chance to think on the bright side. That is to say, we now get to do some experimenting and see how far that takes us in an attempt to fit difficult functions while being able to generalize.

``````%matplotlib notebook

import numpy as np
import matplotlib.pyplot as plt

from utils.analysis import get_weights, predict_targets
from utils.data import create_data, cubic_function, sin_function, quartic_function

from models.custom import create_custom_model, create_periodic_model
from models.callbacks import CustomLRScheduler

``````
``````fig = plt.figure(figsize=(8, 8), facecolor='white')
plt.ion()

fig.show()
fig.canvas.draw()
``````
``````def train_inner(model, weight, x_train, y_train, x_test, y_test, callbacks):
loss="mean_squared_error",
metrics=["mean_squared_error"])

layer = model.get_layer("periodic")
layer.set_trainable_weight(weight)

model.fit(x_train, y_train, epochs=25, validation_data=[x_test, y_test], verbose=False, callbacks=callbacks)
print(layer.get_weights())

pred = model.predict(x_train)
ax.clear()
ax.plot(x_train.reshape(-1), y_train.reshape(-1), color="b")
ax.plot(x_train.reshape(-1), pred.reshape(-1), color="r")

fig.canvas.draw()

return pred

def train_outer(model, epochs, x_train, y_train, x_test, y_test, callbacks):
weights = ["periodic_amplitude", "periodic_shift", "periodic_periodicity"]
preds = []

for i in range(epochs):
print("Epoch: {}".format(i))

for weight in weights:
print("Updating weight: {}".format(weight))
pred = train_inner(model, weight, x_train, y_train, x_test, y_test, callbacks)
preds.append(pred)

return model, preds

def train_basic(model, epochs, x_train, y_train, x_test, y_test):
loss="mean_squared_error",
metrics=["mean_squared_error"])
preds = []

for i in range(epochs):
model.fit(x_train, y_train, epochs=25, validation_data=[x_test, y_test], verbose=False)
pred = model.predict(x_train)
preds.append(pred)

return model, preds
``````
``````x_train, y_train, x_test, y_test = create_data(-5, 5, -7, 7, 1000, sin_function, [4, 20, 0.5, 0], add_noise=False)

lr_scheduler = [CustomLRScheduler()]
model = create_periodic_model(0.5, 0.00005)
model_iterative, preds_iterative = train_outer(model, 10, x_train, y_train, x_test, y_test, lr_scheduler)

model = create_periodic_model(0.5, 0.00005)
model_all, preds_all = train_basic(model, 10, x_train, y_train, x_test, y_test)
``````
``````from utils.plot import animated_training_plot
animated_training_plot(x_train, y_train, preds_iterative, "training_clip_iterative.gif", fps=15)
animated_training_plot(x_train, y_train, preds_all, "training_clip_all.gif", fps=15)
``````
``````
``````