import torch
print(torch.__version__)

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# to avoid writing torch.nn
import torch.nn as nn

X = torch.linspace(1,50,50).reshape(-1,1)

torch.manual_seed(71) # to obtain reproducible results
e = torch.randint(-8,9,(50,1),dtype=torch.float)

# adding noise to make our data noisy for modeling
y = 2*X + 1 + e
print(f"X shape is {X.shape} and y shape is {y.shape}")

# we cannot plot tensor from torch so we need to convert it to numpy array
plt.figure(figsize=(10, 7))
plt.scatter(X.numpy(), y.numpy())
plt.xlabel('X')
plt.ylabel('y')
plt.show();

# To set a seed in order to get exact same value everytime we run the cell
torch.manual_seed(59)

model = nn.Linear(in_features=1, out_features=1)


print(model.weight)
print(model.bias)

class Model(nn.Module):
    def __init__(self, in_features, out_features):
        super().__init__()
        self.linear = nn.Linear(in_features, out_features)

    def forward(self, x):
        y_pred = self.linear(x)
        return y_pred

torch.manual_seed(59)
model = Model(1, 1)
print(model)
print('Weight:', model.linear.weight.item())
print('Bias:  ', model.linear.bias.item())

for name, param in model.named_parameters():
    print(name, '\t', param.item())

criterion = nn.MSELoss()

optimizer = torch.optim.SGD(model.parameters(), lr= 0.001)

# Set a reasonably large number of passes
epochs = 50

# Create a list to store loss values. This will let us view our progress afterward.
losses = []

for i in range(epochs):
    i += 1

    # Create a prediction set by running "X" through the current model parameters
    y_pred = model.forward(X)

    # claculating loss function
    loss = criterion(y_pred, y)

    # Add the loss value to our tracking list
    # as loss coming from criterion we need to detach it and converted it to numpy
    # array in order to plot it later
    losses.append(loss.detach().numpy())

    # Print the current line of results
    print(f'epoch: {i:2}  loss: {loss.item():10.8f} \
          weight: {model.linear.weight.item():10.8f}  \
          bias: {model.linear.bias.item():10.8f}')

    # Gradients accumulate with every backprop. To prevent compounding we need
    # to reset the stored gradient for each new epoch.
    optimizer.zero_grad()

    # Now we can backprop
    loss.backward()

    # Finally, we can update the hyperparameters of our model
    optimizer.step()

plt.plot(range(epochs), losses)
plt.ylabel('Loss')
plt.xlabel('epoch');

w1,b1 = model.linear.weight.item(), model.linear.bias.item()
print(f'Current weight: {w1:.8f}, Current bias: {b1:.8f}')
print()

x1 = np.array([X.min(),X.max()])

y1 = x1*w1 + b1

plt.figure(figsize=(10, 7))

plt.scatter(X.numpy(), y.numpy(), label = "Actual")
plt.plot(x1,y1,'r', label = "Prediction")
plt.title('Initial Model')
plt.ylabel('y')
plt.xlabel('x')
plt.legend()
plt.show();

import pandas as pd

from sklearn.datasets import load_iris

iris = load_iris()
df = pd.DataFrame(data= np.c_[iris['data'], iris['target']],
                     columns= iris['feature_names'] + ['target'])

df.head()

fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(10,7))
fig.tight_layout()

plots = [(0,1),(2,3),(0,2),(1,3)]
colors = ['b', 'r', 'g']
labels = ['Iris setosa','Iris virginica','Iris versicolor']

for i, ax in enumerate(axes.flat):
    for j in range(3):
        x = df.columns[plots[i][0]]
        y = df.columns[plots[i][1]]
        ax.scatter(df[df['target']==j][x], df[df['target']==j][y], color=colors[j])
        ax.set(xlabel=x, ylabel=y)

fig.legend(labels=labels, loc=3, bbox_to_anchor=(1.0,0.85))
plt.show()

from sklearn.model_selection import train_test_split

train_X, test_X, train_y, test_y = train_test_split(df.drop('target',axis=1).values,
                                                    df['target'].values, test_size=0.2,
                                                    random_state=33)

# Casting traina and test data set to pytorch tensor
X_train = torch.FloatTensor(train_X)
X_test = torch.FloatTensor(test_X)
# y_train = F.one_hot(torch.LongTensor(y_train))  # not needed with Cross Entropy Loss
# y_test = F.one_hot(torch.LongTensor(y_test))
y_train = torch.LongTensor(train_y)
y_test = torch.LongTensor(test_y)

print(f'Training size: {len(y_train)}')
labels, counts = y_train.unique(return_counts=True)
print(f'Labels: {labels}\nCounts: {counts}')

from torch.utils.data import TensorDataset, DataLoader

# Data comming from all columns except target, so we are droping "target"
data = df.drop('target',axis=1).values
labels = df['target'].values

# TensorDataset to split train and test
iris = TensorDataset(torch.FloatTensor(data),torch.LongTensor(labels))

type(iris)

iris_loader = DataLoader(iris, batch_size=32, shuffle=True)

trainloader = DataLoader(X_train, batch_size=60, shuffle=True)

testloader = DataLoader(X_test, batch_size=60, shuffle=False)

import torch.nn.functional as F

from pyparsing.helpers import Forward

class Model(nn.Module):

  def __init__(self, in_features = 4, h1 = 8, h2 = 9, out_features = 3):
      # how many layers?
      # input layer (4 feature) --> h1 N ---> h2 N ---> output(3 classes)
      super().__init__()
      self.fc1 = nn.Linear(in_features, h1)   # input layer
      self.fc2 = nn.Linear(h1, h2)            # hidden layer
      self.out = nn.Linear(h2, out_features)  # output layer

  def forward(self, x):

    x = F.relu(self.fc1(x))
    x = F.relu(self.fc2(x))
    x = self.out(x)

    return x

# Instantiate the Model class using parameter defaults:

torch.manual_seed(32)
model = Model()

# in PyTorh you don't need to perform oneHotEncoder as CrossEntropyLoss perform this for us

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr = 0.01)

epochs = 100
losses = []

for i in range(epochs):
    i += 1
    y_pred = model.forward(X_train)
    loss = criterion(y_pred, y_train)
    losses.append(loss.detach().numpy())


    # a neat trick to save screen space:
    # print only every 10 epochs
    if i % 10 == 1:
        print(f'epoch: {i:2}  loss: {loss.item():10.8f}')

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

plt.plot(range(epochs), losses)
plt.ylabel('Loss')
plt.xlabel('epoch');

correct = 0
# no_grad() say pytorch we are not training based on this data and it's only evaluation
with torch.no_grad():
    for i,data in enumerate(X_test):
        y_val = model.forward(data)
        print(f'{i+1:2}. {str(y_val):38}  {y_test[i]}')
        if y_val.argmax().item() == y_test[i]:
            correct += 1
print(f'\n{correct} out of {len(y_test)} = {100*correct/len(y_test):.2f}% correct')

# Saving data
torch.save(model.state_dict(), 'IrisDatasetModel.pt')

new_model = Model()
new_model.load_state_dict(torch.load('IrisDatasetModel.pt'))
new_model.eval()

import torch
import torch.nn as nn

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline

df = pd.read_csv('https://raw.githubusercontent.com/arminnorouzi/machine_learning_course_UofA_MECE610/main/Data/NYCTaxiFares.csv')
df.head()

df['fare_amount'].describe()

def haversine_distance(df, lat1, long1, lat2, long2):
    """Calculates the haversine distance between 2 sets of GPS coordinates in df

    Arg:
        lat1 (str): latitudes column name of point 1 in df
        lat2 (str): latitudes column name of point 2 in df
        long1 (str): longitudes column name of point 1 in df
        long2 (str): longitudes column name of point 2 in df

    return:
        d (float): Traveled distance
    """
    r = 6371  # average radius of Earth in kilometers

    phi1 = np.radians(df[lat1])
    phi2 = np.radians(df[lat2])

    # Converting to radians
    delta_phi = np.radians(df[lat2] - df[lat1])
    delta_lambda = np.radians(df[long2] - df[long1])

    a = np.sin(delta_phi / 2)**2 + np.cos(phi1) * np.cos(phi2) * np.sin(delta_lambda / 2)**2
    c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
    d = (r * c) # in kilometers

    return d

df['dist_km'] = haversine_distance(df,'pickup_latitude', 'pickup_longitude', 'dropoff_latitude', 'dropoff_longitude')
df.head()

df.info()

# Change time to EST from UTC --> EDT: Eastern Day Time
df['EDTdate'] = pd.to_datetime(df['pickup_datetime'].str[:19]) - pd.Timedelta(hours = 4)
df['Hour'] = df['EDTdate'].dt.hour
df['AMorPM'] = np.where(df['Hour'] < 12,'am','pm')
df['Weekday'] = df['EDTdate'].dt.strftime("%a")
# df['Weekday'] = df['EDTdate'].dt.dayofweek # Give the integer of day of the week
df.head()

df.columns

# categorical columns
cat_cols = ['Hour', 'AMorPM', 'Weekday']

# continuous columns
cont_cols = ['pickup_latitude', 'pickup_longitude', 'dropoff_latitude',
             'dropoff_longitude', 'passenger_count', 'dist_km']

# this column contains the labels - continuous as we are working on regression problem
y_col = ['fare_amount']

# Convert our three categorical columns to category dtypes.
for cat in cat_cols:
    df[cat] = df[cat].astype('category')

df['AMorPM']

df['AMorPM'].cat.categories

df['AMorPM'].cat.codes

hr = df['Hour'].cat.codes.values
ampm = df['AMorPM'].cat.codes.values
wkdy = df['Weekday'].cat.codes.values

# stacking hr ampm wkdy
cats = np.stack([hr, ampm, wkdy], 1)

# printing first 5 elements
cats[:5]

# Convert categorical variables to a tensor
cats = torch.tensor(cats, dtype=torch.int64)
# this syntax is ok, since the source data is an array, not an existing tensor

cats[:5]

# Convert continuous variables to a tensor
conts = np.stack([df[col].values for col in cont_cols], 1)
conts = torch.tensor(conts, dtype=torch.float)
conts[:5]

# Convert labels to a tensor
y = torch.tensor(df[y_col].values, dtype=torch.float).reshape(-1,1)

y[:5]

print(f" Catagorical data {cats.shape}, continuous data {conts.shape}, and label size {y.shape}")

# This will set embedding sizes for Hours, AMvsPM and Weekdays
cat_szs = [len(df[col].cat.categories) for col in cat_cols]
emb_szs = [(size, min(50, (size+1)//2)) for size in cat_szs]
emb_szs

# This is our source data
catz = cats[:4]
catz

# This is assigned inside the __init__() method
selfembeds = nn.ModuleList([nn.Embedding(ni, nf) for ni,nf in emb_szs])
selfembeds

list(enumerate(selfembeds))

# This happens inside the forward() method
embeddingz = []
for i,e in enumerate(selfembeds):
    embeddingz.append(e(catz[:,i]))
embeddingz

# We concatenate the embedding sections (12,1,4) into one (17)
z = torch.cat(embeddingz, 1)
z

# This was assigned under the __init__() method
selfembdrop = nn.Dropout(.4)
z = selfembdrop(z)
z

class TabularModel(nn.Module):

    def __init__(self, emb_szs, n_cont, out_sz, layers, p = 0.5):
        '''Extend the base Module class, set up the following parameters:

            emb_szs (list of tuples): each categorical variable size is paired with an embedding size
            n_cont (int): number of continuous variables
            out_sz (int): output size
            layers (list of ints): layer sizes
            p (float) dropout probability for each layer (for simplicity we'll use the same value throughout)
        '''
        super().__init__()

        # Set up the embedded layers with torch.nn.ModuleList() and torch.nn.Embedding()
        # Categorical data will be filtered through these Embeddings in the forward section.
        self.embeds = nn.ModuleList([nn.Embedding(ni, nf) for ni,nf in emb_szs])

        # Set up a dropout function for the embeddings with torch.nn.Dropout() The default p-value=0.5
        self.emb_drop = nn.Dropout(p)

        # Set up a normalization function for the continuous variables with torch.nn.BatchNorm1d()
        self.bn_cont = nn.BatchNorm1d(n_cont)

        # Set up a sequence of neural network layers where each level includes a Linear function,
        # an activation function (we'll use ReLU), a normalization step, and a dropout layer.
        layerlist = []
        n_emb = sum((nf for ni,nf in emb_szs))
        n_in = n_emb + n_cont

        for i in layers:
            layerlist.append(nn.Linear(n_in,i))
            layerlist.append(nn.ReLU(inplace=True))
            layerlist.append(nn.BatchNorm1d(i))
            layerlist.append(nn.Dropout(p))
            n_in = i
        layerlist.append(nn.Linear(layers[-1],out_sz))

        # We'll combine the list of layers with torch.nn.Sequential()
        self.layers = nn.Sequential(*layerlist)

    def forward(self, x_cat, x_cont):
        # Define the forward method. Preprocess the embeddings and normalize the continuous variables
        # before passing them through the layers.
        # Use torch.cat() to combine multiple tensors into one.
        embeddings = []
        for i,e in enumerate(self.embeds):
            embeddings.append(e(x_cat[:,i]))
        x = torch.cat(embeddings, 1)
        x = self.emb_drop(x)

        # combining catagorical and continuouse
        x_cont = self.bn_cont(x_cont)
        x = torch.cat([x, x_cont], 1)
        x = self.layers(x)
        return x

torch.manual_seed(33)
model = TabularModel(emb_szs, conts.shape[1], 1, [200,100], p=0.4)
model

criterion = nn.MSELoss()  # we'll convert this to RMSE later
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

batch_size = 60000
test_size = int(batch_size * .2)


cat_train = cats[:batch_size-test_size]
cat_test = cats[batch_size-test_size:batch_size]
con_train = conts[:batch_size-test_size]
con_test = conts[batch_size-test_size:batch_size]
y_train = y[:batch_size-test_size]
y_test = y[batch_size-test_size:batch_size]

print(f"Size of cat size data {len(cat_train)} and Size of cat size data {len(con_train)}")

import time
start_time = time.time()

epochs = 300
losses = []

for i in range(epochs):
    i += 1
    y_pred = model(cat_train, con_train)
    loss = torch.sqrt(criterion(y_pred, y_train)) # RMSE
    losses.append(loss.detach().numpy())

    # a neat trick to save screen space:
    if i % 25 == 1:
        print(f'epoch: {i:3}  loss: {loss.item():10.8f}')

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

print(f'epoch: {i:3}  loss: {loss.item():10.8f}') # print the last line
print(f'\nDuration: {time.time() - start_time:.0f} seconds') # print the time elapsed

plt.plot(range(epochs), losses)
plt.ylabel('RMSE Loss')
plt.xlabel('epoch');

# TO EVALUATE THE ENTIRE TEST SET
with torch.no_grad():
    y_val = model(cat_test, con_test)
    loss = torch.sqrt(criterion(y_val, y_test))
print(f'RMSE: {loss:.8f}')

print(f'{"PREDICTED":>12} {"ACTUAL":>8} {"DIFF":>8}')
for i in range(50):
    diff = np.abs(y_val[i].item()-y_test[i].item())
    print(f'{i+1:2}. {y_val[i].item():8.4f} {y_test[i].item():8.4f} {diff:8.4f}')

# Make sure to save the model only after the training has happened!
if len(losses) == epochs:
    torch.save(model.state_dict(), 'TaxiFareRegrModel.pt')
else:
    print('Model has not been trained. Consider loading a trained model instead.')

import torch
import torch.nn as nn
import numpy as np
import pandas as pd

def haversine_distance(df, lat1, long1, lat2, long2):
    r = 6371
    phi1 = np.radians(df[lat1])
    phi2 = np.radians(df[lat2])
    delta_phi = np.radians(df[lat2]-df[lat1])
    delta_lambda = np.radians(df[long2]-df[long1])
    a = np.sin(delta_phi/2)**2 + np.cos(phi1) * np.cos(phi2) * np.sin(delta_lambda/2)**2
    c = 2 * np.arctan2(np.sqrt(a), np.sqrt(1-a))
    return r * c

class TabularModel(nn.Module):
    def __init__(self, emb_szs, n_cont, out_sz, layers, p=0.5):
        super().__init__()
        self.embeds = nn.ModuleList([nn.Embedding(ni, nf) for ni,nf in emb_szs])
        self.emb_drop = nn.Dropout(p)
        self.bn_cont = nn.BatchNorm1d(n_cont)
        layerlist = []
        n_emb = sum((nf for ni,nf in emb_szs))
        n_in = n_emb + n_cont
        for i in layers:
            layerlist.append(nn.Linear(n_in,i))
            layerlist.append(nn.ReLU(inplace=True))
            layerlist.append(nn.BatchNorm1d(i))
            layerlist.append(nn.Dropout(p))
            n_in = i
        layerlist.append(nn.Linear(layers[-1],out_sz))
        self.layers = nn.Sequential(*layerlist)
    def forward(self, x_cat, x_cont):
        embeddings = []
        for i,e in enumerate(self.embeds):
            embeddings.append(e(x_cat[:,i]))
        x = torch.cat(embeddings, 1)
        x = self.emb_drop(x)
        x_cont = self.bn_cont(x_cont)
        x = torch.cat([x, x_cont], 1)
        return self.layers(x)

emb_szs = [(24, 12), (2, 1), (7, 4)]
model2 = TabularModel(emb_szs, 6, 1, [200,100], p=0.4)

model2.load_state_dict(torch.load('TaxiFareRegrModel.pt'));
model2.eval() # be sure to run this step!

def test_data(mdl): # pass in the name of the new model
    # INPUT NEW DATA
    plat = float(input('What is the pickup latitude?  '))
    plong = float(input('What is the pickup longitude? '))
    dlat = float(input('What is the dropoff latitude?  '))
    dlong = float(input('What is the dropoff longitude? '))
    psngr = int(input('How many passengers? '))
    dt = input('What is the pickup date and time?\nFormat as YYYY-MM-DD HH:MM:SS     ')

    # PREPROCESS THE DATA
    dfx_dict = {'pickup_latitude':plat,'pickup_longitude':plong,'dropoff_latitude':dlat,
         'dropoff_longitude':dlong,'passenger_count':psngr,'EDTdate':dt}
    dfx = pd.DataFrame(dfx_dict, index=[0])
    dfx['dist_km'] = haversine_distance(dfx,'pickup_latitude', 'pickup_longitude',
                                        'dropoff_latitude', 'dropoff_longitude')
    dfx['EDTdate'] = pd.to_datetime(dfx['EDTdate'])

    # We can skip the .astype(category) step since our fields are small,
    # and encode them right away
    dfx['Hour'] = dfx['EDTdate'].dt.hour
    dfx['AMorPM'] = np.where(dfx['Hour']<12,0,1)
    dfx['Weekday'] = dfx['EDTdate'].dt.strftime("%a")
    dfx['Weekday'] = dfx['Weekday'].replace(['Fri','Mon','Sat','Sun','Thu','Tue','Wed'],
                                            [0,1,2,3,4,5,6]).astype('int64')
    # CREATE CAT AND CONT TENSORS
    cat_cols = ['Hour', 'AMorPM', 'Weekday']
    cont_cols = ['pickup_latitude', 'pickup_longitude', 'dropoff_latitude',
                 'dropoff_longitude', 'passenger_count', 'dist_km']
    xcats = np.stack([dfx[col].values for col in cat_cols], 1)
    xcats = torch.tensor(xcats, dtype=torch.int64)
    xconts = np.stack([dfx[col].values for col in cont_cols], 1)
    xconts = torch.tensor(xconts, dtype=torch.float)

    # PASS NEW DATA THROUGH THE MODEL WITHOUT PERFORMING A BACKPROP
    with torch.no_grad():
        z = mdl(xcats, xconts)
    print(f'\nThe predicted fare amount is ${z.item():.2f}')

z = test_data(model2)

torch.manual_seed(33)
model_classification = TabularModel(emb_szs, conts.shape[1], 2, [200,100], p=0.4) # out_sz = 2

model_classification

y_col_classification = ['fare_class']  # this column contains the labels
y_classification = torch.tensor(df[y_col_classification].values).flatten()


batch_size = 60000
test_size = 12000

cat_train_classification = cats[:batch_size-test_size]
cat_test_classification = cats[batch_size-test_size:batch_size]
con_train_classification = conts[:batch_size-test_size]
con_test_classification = conts[batch_size-test_size:batch_size]


y_train_classification = y_classification[:batch_size-test_size]
y_test_classification = y_classification[batch_size-test_size:batch_size]

criterion_classification = nn.CrossEntropyLoss()
optimizer_classification = torch.optim.Adam(model_classification.parameters(), lr=0.001)

import time
start_time = time.time()

epochs = 300
losses_classification = []

for i in range(epochs):
    i += 1
    y_pred = model_classification(cat_train_classification, con_train_classification)
    loss = criterion_classification(y_pred, y_train_classification)
    losses_classification.append(loss.detach().numpy())

    # a neat trick to save screen space:
    if i % 25 == 1:
        print(f'epoch: {i:3}  loss: {loss.item():10.8f}')

    optimizer_classification.zero_grad()
    loss.backward()
    optimizer_classification.step()

print(f'epoch: {i:3}  loss: {loss.item():10.8f}') # print the last line
print(f'\nDuration: {time.time() - start_time:.0f} seconds') # print the time elapsed

plt.plot(range(epochs), losses_classification)
plt.ylabel('Cross Entropy Loss')
plt.xlabel('epoch');

# TO EVALUATE THE ENTIRE TEST SET
with torch.no_grad():
    y_val = model_classification(cat_test_classification, con_test_classification)
    loss = criterion_classification(y_val, y_test_classification)
print(f'CE Loss: {loss:.8f}')

rows = 50
correct = 0
print(f'{"MODEL OUTPUT":26} ARGMAX  Y_TEST')
for i in range(rows):
    print(f'{str(y_val[i]):26} {y_val[i].argmax():^7}{y_test_classification[i]:^7}')
    if y_val[i].argmax().item() == y_test_classification[i]:
        correct += 1
print(f'\n{correct} out of {rows} = {100*correct/rows:.2f}% correct')