# Import Packages
import httplib2
from bs4 import BeautifulSoup, SoupStrainer
import urllib.request
import os

# root directory of the project
root_path = os.path.dirname(os.path.realpath(__file__))

# Declare an http object
http = httplib2.Http()

# Get the status and respose from the webpage
status, response = http.request('https://fangj.github.io/friends/')

# Create a list
links = []

# (same directory) in append mode and 
file1 = open(root_path + "/dataset/dataset.txt","a", encoding="utf-8") 

# Open the webpage, declaring the right decoder and parser
parser = 'html.parser'  # or 'lxml' (preferred) or 'html5lib', if installed
resp = urllib.request.urlopen("https://fangj.github.io/friends/")
soup = BeautifulSoup(resp, parser, from_encoding=resp.info().get_param('charset'))

# Add hyperlinks to a list
for link in soup.find_all('a', href=True):
    links.append("https://fangj.github.io/friends/" + link['href'])

# Open each webpage, read all text, then write the text to a file
for i in range(0, len(links)):
    resp = urllib.request.urlopen(links[i])
    print("Reading " + str(resp))
    soup = BeautifulSoup(resp, features="lxml")
    txt = soup.get_text()
    file1.write(txt)

# Close the file
file1.close()

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    This python script is used as a web scraper to gather all Friends scripts through various links
    found at this website https://fangj.github.io/friends/
"""

# Import Packages
import httplib2
from bs4 import BeautifulSoup, SoupStrainer
import urllib.request
import os

# root directory of the project
root_path = os.path.dirname(os.path.realpath(__file__))

# Declare an http object
http = httplib2.Http()

# Get the status and respose from the webpage
status, response = http.request('https://fangj.github.io/friends/')

# Create a list
links = []

# (same directory) in append mode and 
file1 = open(root_path + "/dataset/dataset.txt","a", encoding="utf-8") 

# Open the webpage, declaring the right decoder and parser
parser = 'html.parser'  # or 'lxml' (preferred) or 'html5lib', if installed
resp = urllib.request.urlopen("https://fangj.github.io/friends/")
soup = BeautifulSoup(resp, parser, from_encoding=resp.info().get_param('charset'))

# Add hyperlinks to a list
for link in soup.find_all('a', href=True):
    links.append("https://fangj.github.io/friends/" + link['href'])

# Open each webpage, read all text, then write the text to a file
for i in range(0, len(links)):
    resp = urllib.request.urlopen(links[i])
    print("Reading " + str(resp))
    soup = BeautifulSoup(resp, features="lxml")
    txt = soup.get_text()
    file1.write(txt)

# Close the file
file1.close()

# Import necessary libraries
import tensorflow as tf
import numpy as np
import os
import time

# These are for generating a random logs folder
import string
import random

# Root directory of the project
root_path = os.path.dirname(os.path.realpath(__file__))

#Allow GPU Growth
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
  try:
    for gpu in gpus:
      tf.config.experimental.set_memory_growth(gpu, True)
  except RuntimeError as e:
    print(e)

############################################################
#  Settings
############################################################
# The maximum length sentence we want for a single input in characters
seq_length = 100

# Number of RNN units
rnn_units = 1024

# The embedding dimension
embedding_dim = 256

# Batch size
BATCH_SIZE = 65

# Number of epochs to run through
EPOCHS=10

BUFFER_SIZE = 100

# Steps per epochs
STEPS = 50

# Low temperatures result in more predictable text.
# Higher temperatures result in more surprising text.
# Experiment to find the best setting.
temperature = 1.0

# Number of characters to generate
num_generate = 200

# Initialization of loss value as a global variable
# to be used in multiple functions
loss = 0

  # Path to dataset
  dataset_path = os.path.join(root_path, r"dataset/dataset.txt")

  # Read, then decode for py2 compat.
  text = open(dataset_path, 'rb').read().decode(encoding='unicode_escape')

  # The unique characters in the file
  vocab = sorted(set(text))

  # Creating a mapping from unique characters to indices
  char2idx = {u:i for i, u in enumerate(vocab)}
  idx2char = np.array(vocab)

  text_as_int = np.array([char2idx[c] for c in text])

  # Create training examples / targets
  char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)

  # Create the sequence from the dataset
  sequences = char_dataset.batch(seq_length+1, drop_remainder=True)

  def split_input_target(chunk):
    input_text = chunk[:-1]
    target_text = chunk[1:]
    return input_text, target_text
   
  # Split the dataset up for better readability and passing into epochs and batch sizes
  dataset = sequences.map(split_input_target)

############################################################
#  Create Dataset
############################################################

def create_dataset():
  # Path to dataset
  dataset_path = os.path.join(root_path, r"dataset/dataset.txt")

  # Read, then decode for py2 compat.
  text = open(dataset_path, 'rb').read().decode(encoding='unicode_escape')

  # The unique characters in the file
  vocab = sorted(set(text))

  # Creating a mapping from unique characters to indices
  char2idx = {u:i for i, u in enumerate(vocab)}
  idx2char = np.array(vocab)

  text_as_int = np.array([char2idx[c] for c in text])

  # Create training examples / targets
  char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)

  # Create the sequence from the dataset
  sequences = char_dataset.batch(seq_length+1, drop_remainder=True)

  def split_input_target(chunk):
    input_text = chunk[:-1]
    target_text = chunk[1:]
    return input_text, target_text
   
  # Split the dataset up for better readability and passing into epochs and batch sizes
  dataset = sequences.map(split_input_target)

  return dataset, vocab, idx2char, char2idx

############################################################
#  Model
############################################################

def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
  model = tf.keras.Sequential([
    tf.keras.layers.Embedding(vocab_size, embedding_dim,
                              batch_input_shape=[batch_size, None]),
    tf.keras.layers.GRU(rnn_units,
                        return_sequences=True,
                        stateful=True,
                        recurrent_initializer='glorot_uniform'),
    tf.keras.layers.Dense(vocab_size)
  ])
  return model

  dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)

  # Length of the vocabulary in chars
  vocab_size = len(vocab)

  model = build_model(
    vocab_size = vocab_size,
    embedding_dim=embedding_dim,
    rnn_units=rnn_units,
    batch_size=BATCH_SIZE)

  for input_example_batch, target_example_batch in dataset.take(1):
    example_batch_predictions = model(input_example_batch)
    print(example_batch_predictions.shape, "# (batch_size, sequence_length, vocab_size)")

  # Print out the different layers of the model
  model.summary()

  sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)
  sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()

  sampled_indices

  def loss(labels, logits):
    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)

  model.compile(optimizer='adam', loss=loss)

  # generate a random string 15 characters long
  random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(15))

  # Directory where the checkpoints will be saved
  checkpoint_dir = './logs/' + random_string + "/"

  # Name of the checkpoint files
  checkpoint_prefix = os.path.join(checkpoint_dir, "Model_{epoch}")

  checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(
      filepath=checkpoint_prefix,
      save_weights_only=True,
      save_freq='epoch')

  model.save_weights(checkpoint_prefix.format(epoch=0))

  history = model.fit(dataset, steps_per_epoch=STEPS, epochs=EPOCHS, callbacks=[checkpoint_callback])

  # Create a path for the saving location of the model
  model_dir = checkpoint_dir + "model.h5"

  # Save the model
  # TODO: Known issue with saving the model and loading it back
  # later in the script causes issues. Working to fix this issue
  model.save(model_dir)

  # Train from the last checkpoint
  tf.train.latest_checkpoint(checkpoint_dir)

  # Build the model with the dataset generated earlier
  model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)

  # Load the weights
  model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))

  # Build the model
  model.build(tf.TensorShape([1, None]))

  # Print out the model summary
  model.summary()

############################################################
#  Train Model
############################################################

def train_model(dataset, vocab):
  # Buffer size to shuffle the dataset
  # (TF data is designed to work with possibly infinite sequences,
  # so it doesn't attempt to shuffle the entire sequence in memory. Instead,
  # it maintains a buffer in which it shuffles elements).
  dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)

  # Length of the vocabulary in chars
  vocab_size = len(vocab)

  model = build_model(
    vocab_size = vocab_size,
    embedding_dim=embedding_dim,
    rnn_units=rnn_units,
    batch_size=BATCH_SIZE)

  for input_example_batch, target_example_batch in dataset.take(1):
    example_batch_predictions = model(input_example_batch)
    print(example_batch_predictions.shape, "# (batch_size, sequence_length, vocab_size)")

  # Print out the different layers of the model
  model.summary()

  sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)
  sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()

  sampled_indices

  def loss(labels, logits):
    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)

  example_batch_loss  = loss(target_example_batch, example_batch_predictions)
  print("Prediction shape: ", example_batch_predictions.shape, " # (batch_size, sequence_length, vocab_size)")

  model.compile(optimizer='adam', loss=loss)

  # generate a random string 15 characters long
  random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(15))

  # Directory where the checkpoints will be saved
  checkpoint_dir = './logs/' + random_string + "/"

  # Name of the checkpoint files
  checkpoint_prefix = os.path.join(checkpoint_dir, "Model_{epoch}")

  checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(
      filepath=checkpoint_prefix,
      save_weights_only=True,
      save_freq='epoch')

  model.save_weights(checkpoint_prefix.format(epoch=0))

  history = model.fit(dataset, steps_per_epoch=STEPS, epochs=EPOCHS, callbacks=[checkpoint_callback])

  # Create a path for the saving location of the model
  model_dir = checkpoint_dir + "model.h5"

  # Save the model
  # TODO: Known issue with saving the model and loading it back
  # later in the script causes issues. Working to fix this issue
  model.save(model_dir)

  # Train from the last checkpoint
  tf.train.latest_checkpoint(checkpoint_dir)

  # Build the model with the dataset generated earlier
  model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)

  # Load the weights
  model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))

  # Build the model
  model.build(tf.TensorShape([1, None]))

  # Print out the model summary
  model.summary()

  # Return the model
  return model

############################################################
#  Generate Text
############################################################

def generate_text(model, idx2char, char2idx, start_string):
  # Evaluation step (generating text using the learned model)

  # Converting our start string to numbers (vectorizing)
  input_eval = [char2idx[s] for s in start_string]
  input_eval = tf.expand_dims(input_eval, 0)

  # Empty array to store our results
  text_generated = []

  # Here batch size == 1
  model.reset_states()
  for i in range(num_generate):
      predictions = model(input_eval)
      # remove the batch dimension
      predictions = tf.squeeze(predictions, 0)

      # using a categorical distribution to predict the character returned by the model
      predictions = predictions / temperature
      predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()

      # We pass the predicted character as the next input to the model
      # along with the previous hidden state
      input_eval = tf.expand_dims([predicted_id], 0)

      text_generated.append(idx2char[predicted_id])

  return (start_string + ''.join(text_generated))

############################################################
#  Configure
############################################################

if __name__ == '__main__':
    import argparse

    # Parse command line arguments
    parser = argparse.ArgumentParser(
        description='Train or Detect.')
    parser.add_argument("command",
                        metavar="<command>",
                        help="'train' or 'generate'")
    parser.add_argument('--weights', required=False,
                        metavar="/path/to/weights",
                        help="Path to weights file")
    parser.add_argument('--start', required=False,
                        metavar="start of string",
                        help="The word that will begin the output string")

    args = parser.parse_args()

    # Configurations
    if args.command == "train":
      # Load in the dataset and other function to use
      dataset, vocab, idx2char, char2idx = create_dataset()
      # Train the model
      model = train_model(dataset, vocab)
      print(generate_text(model, idx2char, char2idx,start_string=u"The "))
    
    if args.command == 'generate':
      # Load in the dataset and other function to use
      dataset, vocab, idx2char, char2idx = create_dataset()
      # Get the model path
      model_path = os.path.join(root_path, args.weights)
      # Length of the vocabulary in chars
      vocab_size = len(vocab)
      # Build the model with the dataset generated earlier
      model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)
      # Load the weights
      model.load_weights(model_path)
      # Build the model
      model.build(tf.TensorShape([1, None]))
      # Generate text
      print(generate_text(model, idx2char, char2idx,start_string=u"The "))

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
   The purpose of this python script is to both experiment with new deep learning techniques,
   and to further my knowledge on the subject of deep learning.

   I used this as a reference guide:
   https://www.tensorflow.org/tutorials/text/text_generation
"""

# Import necessary libraries
import tensorflow as tf
import numpy as np
import os
import time

# These are for generating a random logs folder
import string
import random

# Root directory of the project
root_path = os.path.dirname(os.path.realpath(__file__))

#Allow GPU Growth
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
  try:
    for gpu in gpus:
      tf.config.experimental.set_memory_growth(gpu, True)
  except RuntimeError as e:
    print(e)

############################################################
#  Settings
############################################################
# The maximum length sentence we want for a single input in characters
seq_length = 100

# Number of RNN units
rnn_units = 1024

# The embedding dimension
embedding_dim = 256

# Batch size
BATCH_SIZE = 65

# Number of epochs to run through
EPOCHS=10

BUFFER_SIZE = 100

# Steps per epochs
STEPS = 50

# Low temperatures result in more predictable text.
# Higher temperatures result in more surprising text.
# Experiment to find the best setting.
temperature = 1.0

# Number of characters to generate
num_generate = 200

# Initialization of loss value as a global variable
# to be used in multiple functions
loss = 0

############################################################
#  Create Dataset
############################################################

def create_dataset():
  # Path to dataset
  dataset_path = os.path.join(root_path, r"dataset/dataset.txt")

  # Read, then decode for py2 compat.
  text = open(dataset_path, 'rb').read().decode(encoding='unicode_escape')

  # The unique characters in the file
  vocab = sorted(set(text))

  # Creating a mapping from unique characters to indices
  char2idx = {u:i for i, u in enumerate(vocab)}
  idx2char = np.array(vocab)

  text_as_int = np.array([char2idx[c] for c in text])

  # Create training examples / targets
  char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)

  # Create the sequence from the dataset
  sequences = char_dataset.batch(seq_length+1, drop_remainder=True)

  def split_input_target(chunk):
    input_text = chunk[:-1]
    target_text = chunk[1:]
    return input_text, target_text
   
  # Split the dataset up for better readability and passing into epochs and batch sizes
  dataset = sequences.map(split_input_target)

  return dataset, vocab, idx2char, char2idx

############################################################
#  Model
############################################################

def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
  model = tf.keras.Sequential([
    tf.keras.layers.Embedding(vocab_size, embedding_dim,
                              batch_input_shape=[batch_size, None]),
    tf.keras.layers.GRU(rnn_units,
                        return_sequences=True,
                        stateful=True,
                        recurrent_initializer='glorot_uniform'),
    tf.keras.layers.Dense(vocab_size)
  ])
  return model

############################################################
#  Train Model
############################################################

def train_model(dataset, vocab):
  # Buffer size to shuffle the dataset
  # (TF data is designed to work with possibly infinite sequences,
  # so it doesn't attempt to shuffle the entire sequence in memory. Instead,
  # it maintains a buffer in which it shuffles elements).
  dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)

  # Length of the vocabulary in chars
  vocab_size = len(vocab)

  model = build_model(
    vocab_size = vocab_size,
    embedding_dim=embedding_dim,
    rnn_units=rnn_units,
    batch_size=BATCH_SIZE)

  for input_example_batch, target_example_batch in dataset.take(1):
    example_batch_predictions = model(input_example_batch)
    print(example_batch_predictions.shape, "# (batch_size, sequence_length, vocab_size)")

  # Print out the different layers of the model
  model.summary()

  sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)
  sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()

  sampled_indices

  def loss(labels, logits):
    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)

  example_batch_loss  = loss(target_example_batch, example_batch_predictions)
  print("Prediction shape: ", example_batch_predictions.shape, " # (batch_size, sequence_length, vocab_size)")

  model.compile(optimizer='adam', loss=loss)

  # generate a random string 15 characters long
  random_string = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(15))

  # Directory where the checkpoints will be saved
  checkpoint_dir = './logs/' + random_string + "/"

  # Name of the checkpoint files
  checkpoint_prefix = os.path.join(checkpoint_dir, "Model_{epoch}")

  checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(
      filepath=checkpoint_prefix,
      save_weights_only=True,
      save_freq='epoch')

  model.save_weights(checkpoint_prefix.format(epoch=0))

  history = model.fit(dataset, steps_per_epoch=STEPS, epochs=EPOCHS, callbacks=[checkpoint_callback])

  # Create a path for the saving location of the model
  model_dir = checkpoint_dir + "model.h5"

  # Save the model
  # TODO: Known issue with saving the model and loading it back
  # later in the script causes issues. Working to fix this issue
  model.save(model_dir)

  # Train from the last checkpoint
  tf.train.latest_checkpoint(checkpoint_dir)

  # Build the model with the dataset generated earlier
  model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)

  # Load the weights
  model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))

  # Build the model
  model.build(tf.TensorShape([1, None]))

  # Print out the model summary
  model.summary()

  # Return the model
  return model

############################################################
#  Generate Text
############################################################

def generate_text(model, idx2char, char2idx, start_string):
  # Evaluation step (generating text using the learned model)

  # Converting our start string to numbers (vectorizing)
  input_eval = [char2idx[s] for s in start_string]
  input_eval = tf.expand_dims(input_eval, 0)

  # Empty array to store our results
  text_generated = []

  # Here batch size == 1
  model.reset_states()
  for i in range(num_generate):
      predictions = model(input_eval)
      # remove the batch dimension
      predictions = tf.squeeze(predictions, 0)

      # using a categorical distribution to predict the character returned by the model
      predictions = predictions / temperature
      predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()

      # We pass the predicted character as the next input to the model
      # along with the previous hidden state
      input_eval = tf.expand_dims([predicted_id], 0)

      text_generated.append(idx2char[predicted_id])

  return (start_string + ''.join(text_generated))

############################################################
#  Configure
############################################################

if __name__ == '__main__':
    import argparse

    # Parse command line arguments
    parser = argparse.ArgumentParser(
        description='Train or Detect.')
    parser.add_argument("command",
                        metavar="<command>",
                        help="'train' or 'generate'")
    parser.add_argument('--weights', required=False,
                        metavar="/path/to/weights",
                        help="Path to weights file")
    parser.add_argument('--start', required=False,
                        metavar="start of string",
                        help="The word that will begin the output string")

    args = parser.parse_args()

    # Configurations
    if args.command == "train":
      # Load in the dataset and other function to use
      dataset, vocab, idx2char, char2idx = create_dataset()
      # Train the model
      model = train_model(dataset, vocab)
      print(generate_text(model, idx2char, char2idx,start_string=u"The "))
    
    if args.command == 'generate':
      # Load in the dataset and other function to use
      dataset, vocab, idx2char, char2idx = create_dataset()
      # Get the model path
      model_path = os.path.join(root_path, args.weights)
      # Length of the vocabulary in chars
      vocab_size = len(vocab)
      # Build the model with the dataset generated earlier
      model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)
      # Load the weights
      model.load_weights(model_path)
      # Build the model
      model.build(tf.TensorShape([1, None]))
      # Generate text
      print(generate_text(model, idx2char, char2idx,start_string=u"The "))