Upload 23 files

counsel-chat-master/.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+__pycache__
+.zip
+.idea

counsel-chat-master/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 nbertagnolli
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

counsel-chat-master/Makefile

@@ -0,0 +1,31 @@
+# Interact Variables
+# defaults
+CHECKPOINT_DIR = "counselchat-convai"
+DATA = "data/counsel_chat_250-tokens_full.json"
+TEMPERATURE = 0.7
+MAX_LENGTH = 30
+MAX_HISTORY = 0
+
+# HELP
+# This will output the help for each task
+# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+.PHONY: help
+
+help: ## This help.
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+.DEFAULT_GOAL := help
+
+
+# DOCKER TASKS
+# Build the container
+build:
+	docker system prune && \
+	docker build -t convai -f docker/Dockerfile .
+
+## Interact with the model
+interact:
+	docker run -it --rm -v $(shell pwd):$(shell pwd) convai:latest \
+	/bin/sh -c 'cd $(shell pwd); \
+	python3 interact.py --model_checkpoint $(CHECKPOINT_DIR) --temperature $(TEMPERATURE) --max_length $(MAX_LENGTH) --max_history $(MAX_HISTORY) --dataset_path $(DATA); \
+	'

counsel-chat-master/README.md

@@ -0,0 +1,25 @@
+# counsel-chat
+This repository holds the code for working with data from counselchat.com.   The scarped data are from individiuals seeking assistance from licensed therapists and their associated responses. The goal is to provide a high quality open source dataset of quality counseling responses.
+
+I've recently added the data to [HuggingFace](https://huggingface.co/datasets/nbertagnolli/counsel-chat) so getting the data should be as easy as:
+
+```python
+from datasets import load_dataset
+
+dataset = load_dataset("nbertagnolli/counsel-chat")
+```
+
+There is a larger writeup available on [medium](https://medium.com/towards-data-science/counsel-chat-bootstrapping-high-quality-therapy-data-971b419f33da)
+
+If you use this data in your work please cite the medium article.
+
+```
+@misc{bertagnolli2020counsel,
+  title={Counsel chat: Bootstrapping high-quality therapy data},
+  author={Bertagnolli, Nicolas},
+  year={2020},
+  publisher={Towards Data Science. https://towardsdatascience. com/counsel-chat~…}
+}
+```
+
+

counsel-chat-master/docker/Dockerfile

@@ -0,0 +1,37 @@
+FROM ubuntu:18.04
+
+MAINTAINER Loreto Parisi [email protected]
+
+########################################  BASE SYSTEM
+# set noninteractive installation
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y apt-utils
+RUN apt-get install -y --no-install-recommends \
+    build-essential \
+    pkg-config \
+    tzdata \
+    curl
+
+######################################## PYTHON3
+RUN apt-get install -y \
+    python3 \
+    python3-pip
+
+# set local timezone
+RUN ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime && \
+    dpkg-reconfigure --frontend noninteractive tzdata
+
+# transfer-learning-conv-ai
+ENV PYTHONPATH /usr/local/lib/python3.6
+COPY . .
+COPY docker/requirements.txt /tmp/requirements.txt
+RUN pip3 install --upgrade pip && pip3 install -r /tmp/requirements.txt
+
+# model zoo
+RUN mkdir models && \
+    curl https://s3.amazonaws.com/models.huggingface.co/transfer-learning-chatbot/finetuned_chatbot_gpt.tar.gz > models/finetuned_chatbot_gpt.tar.gz && \
+    cd models/ && \
+    tar -xvzf finetuned_chatbot_gpt.tar.gz && \
+    rm finetuned_chatbot_gpt.tar.gz
+    
+CMD ["bash"]

counsel-chat-master/docker/requirements.txt

@@ -0,0 +1,10 @@
+numpy
+torch
+pytorch-ignite
+transformers==2.5.1
+tensorboardX==1.8
+jupyter
+scikit-learn
+pandas
+dill
+scipy

counsel-chat-master/interact.py

@@ -0,0 +1,258 @@
+# # Copyright (c) 2019-present, HuggingFace Inc.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+import logging
+import json
+import os
+import random
+from argparse import ArgumentParser
+from itertools import chain
+from pprint import pformat
+import tempfile
+import tarfile
+import warnings
+
+import torch
+import torch.nn.functional as F
+from transformers import cached_path
+
+from transformers import OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, GPT2LMHeadModel, \
+    GPT2Tokenizer
+
+HF_FINETUNED_MODEL = "https://s3.amazonaws.com/models.huggingface.co/transfer-learning-chatbot/gpt_personachat_cache.tar.gz"
+SPECIAL_TOKENS = ["<bos>", "<eos>", "<speaker1>", "<speaker2>", "<pad>"]
+ATTR_TO_SPECIAL_TOKEN = {'bos_token': '<bos>', 'eos_token': '<eos>', 'pad_token': '<pad>',
+                         'additional_special_tokens': ['<speaker1>', '<speaker2>']}
+
+
+def download_pretrained_model():
+    """ Download and extract finetuned model from S3 """
+    resolved_archive_file = cached_path(HF_FINETUNED_MODEL)
+    tempdir = tempfile.mkdtemp()
+    with tarfile.open(resolved_archive_file, 'r:gz') as archive:
+        def is_within_directory(directory, target):
+            
+            abs_directory = os.path.abspath(directory)
+            abs_target = os.path.abspath(target)
+        
+            prefix = os.path.commonprefix([abs_directory, abs_target])
+            
+            return prefix == abs_directory
+        
+        def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
+        
+            for member in tar.getmembers():
+                member_path = os.path.join(path, member.name)
+                if not is_within_directory(path, member_path):
+                    raise Exception("Attempted Path Traversal in Tar File")
+        
+            tar.extractall(path, members, numeric_owner=numeric_owner) 
+            
+        
+        safe_extract(archive, tempdir)
+    return tempdir
+
+
+def get_dataset(tokenizer, dataset_path, dataset_cache):
+    """ Get tokenized PERSONACHAT dataset from S3 or cache."""
+    dataset_path = dataset_path
+    dataset_cache = dataset_cache + '_' + type(tokenizer).__name__  # To avoid using GPT cache for GPT-2 and vice-versa
+    if dataset_cache and os.path.isfile(dataset_cache):
+        dataset = torch.load(dataset_cache)
+    else:
+        personachat_file = cached_path(dataset_path)
+        with open(personachat_file, "r", encoding="utf-8") as f:
+            dataset = json.loads(f.read())
+
+        def tokenize(obj):
+            if isinstance(obj, str):
+                return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(obj))
+            if isinstance(obj, dict):
+                return dict((n, tokenize(o)) for n, o in obj.items())
+            return list(tokenize(o) for o in obj)
+        dataset = tokenize(dataset)
+        torch.save(dataset, dataset_cache)
+    return dataset
+
+
+def add_special_tokens_(model, tokenizer):
+    """ Add special tokens to the tokenizer and the model if they have not already been added. """
+    orig_num_tokens = len(tokenizer.encoder)
+    num_added_tokens = tokenizer.add_special_tokens(ATTR_TO_SPECIAL_TOKEN) # doesn't add if they are already there
+    if num_added_tokens > 0:
+        model.resize_token_embeddings(new_num_tokens=orig_num_tokens + num_added_tokens)
+
+
+def build_input_from_segments(persona, history, reply, tokenizer, lm_labels=False, with_eos=True):
+    """ Build a sequence of input from 3 segments: persona, history and last reply. """
+    bos, eos, speaker1, speaker2 = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[:-1])
+    sequence = [[bos] + list(chain(*persona))] + history + [reply + ([eos] if with_eos else [])]
+    sequence = [sequence[0]] + [[speaker2 if (len(sequence)-i) % 2 else speaker1] + s for i, s in enumerate(sequence[1:])]
+    instance = {}
+    instance["input_ids"] = list(chain(*sequence))
+    instance["token_type_ids"] = [speaker2 if i % 2 else speaker1 for i, s in enumerate(sequence) for _ in s]
+    instance["mc_token_ids"] = len(instance["input_ids"]) - 1
+    instance["lm_labels"] = [-100] * len(instance["input_ids"])
+    if lm_labels:
+        instance["lm_labels"] = ([-100] * sum(len(s) for s in sequence[:-1])) + [-100] + sequence[-1][1:]
+    return instance
+
+
+def top_filtering(logits, top_k=0., top_p=0.9, threshold=-float('Inf'),
+                  filter_value=-float('Inf')):
+    """ Filter a distribution of logits using top-k, top-p (nucleus) and/or threshold filtering
+        Args:
+            logits: logits distribution shape (vocabulary size)
+            top_k: <=0: no filtering, >0: keep only top k tokens with highest probability.
+            top_p: <=0.0: no filtering, >0.0: keep only a subset S of candidates, where S is the smallest subset
+                whose total probability mass is greater than or equal to the threshold top_p.
+                In practice, we select the highest probability tokens whose cumulative probability mass exceeds
+                the threshold top_p.
+            threshold: a minimal threshold to keep logits
+    """
+    assert logits.dim() == 1  # Only work for batch size 1 for now - could update but it would obfuscate a bit the code
+    top_k = min(top_k, logits.size(-1))
+    if top_k > 0:
+        # Remove all tokens with a probability less than the last token in the top-k tokens
+        indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
+        logits[indices_to_remove] = filter_value
+
+    if top_p > 0.0:
+        # Compute cumulative probabilities of sorted tokens
+        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
+        cumulative_probabilities = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
+
+        # Remove tokens with cumulative probability above the threshold
+        sorted_indices_to_remove = cumulative_probabilities > top_p
+        # Shift the indices to the right to keep also the first token above the threshold
+        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
+        sorted_indices_to_remove[..., 0] = 0
+
+        # Back to unsorted indices and set them to -infinity
+        indices_to_remove = sorted_indices[sorted_indices_to_remove]
+        logits[indices_to_remove] = filter_value
+
+    indices_to_remove = logits < threshold
+    logits[indices_to_remove] = filter_value
+
+    return logits
+
+
+def sample_sequence(personality, history, tokenizer, model, args, current_output=None):
+    special_tokens_ids = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS)
+    if current_output is None:
+        current_output = []
+
+    for i in range(args.max_length):
+        instance = build_input_from_segments(personality, history, current_output,
+                                             tokenizer, with_eos=False)
+
+        input_ids = torch.tensor(instance["input_ids"], device=args.device).unsqueeze(0)
+        token_type_ids = torch.tensor(instance["token_type_ids"],
+                                      device=args.device).unsqueeze(0)
+
+        logits = model(input_ids, token_type_ids=token_type_ids)
+        if isinstance(logits, tuple):  # for gpt2 and maybe others
+            logits = logits[0]
+        logits = logits[0, -1, :] / args.temperature
+        logits = top_filtering(logits, top_k=args.top_k, top_p=args.top_p)
+        probs = F.softmax(logits, dim=-1)
+
+        prev = torch.topk(probs, 1)[1] if args.no_sample else torch.multinomial(probs, 1)
+        if i < args.min_length and prev.item() in special_tokens_ids:
+            while prev.item() in special_tokens_ids:
+                if probs.max().item() == 1:
+                    warnings.warn(
+                        "Warning: model generating special token with probability 1.")
+                    break  # avoid infinitely looping over special token
+                prev = torch.multinomial(probs, num_samples=1)
+
+        if prev.item() in special_tokens_ids:
+            break
+        current_output.append(prev.item())
+
+    return current_output
+
+
+def run():
+    parser = ArgumentParser()
+    parser.add_argument("--dataset_path", type=str, default="",
+                        help="Path or url of the dataset. If empty download from S3.")
+    parser.add_argument("--dataset_cache", type=str, default='./dataset_cache',
+                        help="Path or url of the dataset cache")
+    parser.add_argument("--model", type=str, default="openai-gpt",
+                        help="Model type (openai-gpt or gpt2)", choices=['openai-gpt',
+                                                                         'gpt2'])  # anything besides gpt2 will load openai-gpt
+    parser.add_argument("--model_checkpoint", type=str, default="",
+                        help="Path, url or short name of the model")
+    parser.add_argument("--max_history", type=int, default=2,
+                        help="Number of previous utterances to keep in history")
+    parser.add_argument("--device", type=str,
+                        default="cuda" if torch.cuda.is_available() else "cpu",
+                        help="Device (cuda or cpu)")
+
+    parser.add_argument("--no_sample", action='store_true',
+                        help="Set to use greedy decoding instead of sampling")
+    parser.add_argument("--max_length", type=int, default=20,
+                        help="Maximum length of the output utterances")
+    parser.add_argument("--min_length", type=int, default=1,
+                        help="Minimum length of the output utterances")
+    parser.add_argument("--seed", type=int, default=0, help="Seed")
+    parser.add_argument("--temperature", type=float, default=0.7,
+                        help="Sampling softmax temperature")
+    parser.add_argument("--top_k", type=int, default=0,
+                        help="Filter top-k tokens before sampling (<=0: no filtering)")
+    parser.add_argument("--top_p", type=float, default=0.9,
+                        help="Nucleus filtering (top-p) before sampling (<=0.0: no filtering)")
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger(__file__)
+    logger.info(pformat(args))
+
+    if args.model_checkpoint == "":
+        if args.model == 'gpt2':
+            raise ValueError(
+                "Interacting with GPT2 requires passing a finetuned model_checkpoint")
+        else:
+            args.model_checkpoint = download_pretrained_model()
+
+    if args.seed != 0:
+        random.seed(args.seed)
+        torch.random.manual_seed(args.seed)
+        torch.cuda.manual_seed(args.seed)
+
+    logger.info("Get pretrained model and tokenizer")
+    tokenizer_class, model_class = (
+    GPT2Tokenizer, GPT2LMHeadModel) if args.model == 'gpt2' else (
+    OpenAIGPTTokenizer, OpenAIGPTLMHeadModel)
+    tokenizer = tokenizer_class.from_pretrained(args.model_checkpoint)
+    model = model_class.from_pretrained(args.model_checkpoint)
+    model.to(args.device)
+    add_special_tokens_(model, tokenizer)
+
+    logger.info("Sample a personality")
+    dataset = get_dataset(tokenizer, args.dataset_path, args.dataset_cache)
+    personalities = [dialog["personality"] for dataset in dataset.values() for dialog in
+                     dataset]
+    personality = random.choice(personalities)
+    logger.info("Selected personality: %s", tokenizer.decode(chain(*personality)))
+
+    history = []
+    while True:
+        raw_text = input(">>> ")
+        while not raw_text:
+            print('Prompt should not be empty!')
+            raw_text = input(">>> ")
+        history.append(tokenizer.encode(raw_text))
+        with torch.no_grad():
+            out_ids = sample_sequence(personality, history, tokenizer, model, args)
+        history.append(out_ids)
+        history = history[-(2 * args.max_history + 1):]
+        out_text = tokenizer.decode(out_ids, skip_special_tokens=True)
+        print(out_text)
+
+
+if __name__ == "__main__":
+    run()

counsel-chat-master/train.py

@@ -0,0 +1,267 @@
+# Copyright (c) 2019-present, HuggingFace Inc.
+# All rights reserved. This source code is licensed under the BSD-style license found in the LICENSE file in the root directory of this source tree.
+import os
+import math
+import logging
+from pprint import pformat
+from argparse import ArgumentParser
+from collections import defaultdict
+from itertools import chain
+
+import torch
+from torch.nn.parallel import DistributedDataParallel
+from torch.utils.data import DataLoader, TensorDataset
+from ignite.engine import Engine, Events
+from ignite.handlers import ModelCheckpoint
+from ignite.metrics import Accuracy, Loss, MetricsLambda, RunningAverage
+from ignite.contrib.handlers import ProgressBar, PiecewiseLinear
+from ignite.contrib.handlers.tensorboard_logger import TensorboardLogger, OutputHandler, OptimizerParamsHandler
+from transformers import (AdamW, OpenAIGPTDoubleHeadsModel, OpenAIGPTTokenizer,
+                                  GPT2DoubleHeadsModel, GPT2Tokenizer, WEIGHTS_NAME, CONFIG_NAME)
+
+from utils import get_dataset, make_logdir
+
+SPECIAL_TOKENS = ["<bos>", "<eos>", "<speaker1>", "<speaker2>", "<pad>"]
+ATTR_TO_SPECIAL_TOKEN = {'bos_token': '<bos>', 'eos_token': '<eos>', 'pad_token': '<pad>',
+                         'additional_special_tokens': ['<speaker1>', '<speaker2>']}
+MODEL_INPUTS = ["input_ids", "mc_token_ids", "lm_labels", "mc_labels", "token_type_ids"]
+PADDED_INPUTS = ["input_ids", "lm_labels", "token_type_ids"]
+
+logger = logging.getLogger(__file__)
+
+def average_distributed_scalar(scalar, args):
+    """ Average a scalar over the nodes if we are in distributed training. We use this for distributed evaluation. """
+    if args.local_rank == -1:
+        return scalar
+    scalar_t = torch.tensor(scalar, dtype=torch.float, device=args.device) / torch.distributed.get_world_size()
+    torch.distributed.all_reduce(scalar_t, op=torch.distributed.ReduceOp.SUM)
+    return scalar_t.item()
+
+
+def pad_dataset(dataset, padding=0):
+    """ Pad the dataset. This could be optimized by defining a Dataset class and padding at the batch level, but this is simpler. """
+    max_l = max(len(x) for x in dataset["input_ids"])
+    for name in PADDED_INPUTS:
+        dataset[name] = [x + [padding if name != "lm_labels" else -100] * (max_l - len(x)) for x in dataset[name]]
+    return dataset
+
+
+def add_special_tokens_(model, tokenizer):
+    """ Add special tokens to the tokenizer and the model if they have not already been added. """
+    orig_num_tokens = len(tokenizer.encoder)
+    num_added_tokens = tokenizer.add_special_tokens(ATTR_TO_SPECIAL_TOKEN) # doesn't add if they are already there
+    if num_added_tokens > 0:
+        model.resize_token_embeddings(new_num_tokens=orig_num_tokens + num_added_tokens)
+
+def build_input_from_segments(persona, history, reply, tokenizer, lm_labels=False, with_eos=True):
+    """ Build a sequence of input from 3 segments: persona, history and last reply. """
+    bos, eos, speaker1, speaker2 = tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[:-1])
+    sequence = [[bos] + list(chain(*persona))] + history + [reply + ([eos] if with_eos else [])]
+    sequence = [sequence[0]] + [[speaker2 if (len(sequence)-i) % 2 else speaker1] + s for i, s in enumerate(sequence[1:])]
+    instance = {}
+    instance["input_ids"] = list(chain(*sequence))
+    instance["token_type_ids"] = [speaker2 if i % 2 else speaker1 for i, s in enumerate(sequence) for _ in s]
+    instance["mc_token_ids"] = len(instance["input_ids"]) - 1
+    instance["lm_labels"] = [-100] * len(instance["input_ids"])
+    if lm_labels:
+        instance["lm_labels"] = ([-100] * sum(len(s) for s in sequence[:-1])) + [-100] + sequence[-1][1:]
+    return instance
+
+
+def get_data_loaders(args, tokenizer):
+    """ Prepare the dataset for training and evaluation """
+    personachat = get_dataset(tokenizer, args.dataset_path, args.dataset_cache)
+
+    logger.info("Build inputs and labels")
+    datasets = {"train": defaultdict(list), "valid": defaultdict(list)}
+    for dataset_name, dataset in personachat.items():
+        num_candidates = len(dataset[0]["utterances"][0]["candidates"])
+        if args.num_candidates > 0 and dataset_name == 'train':
+            num_candidates = min(args.num_candidates, num_candidates)
+        for dialog in dataset:
+            persona = dialog["personality"].copy()
+            for _ in range(args.personality_permutations):
+                for utterance in dialog["utterances"]:
+                    history = utterance["history"][-(2*args.max_history+1):]
+                    for j, candidate in enumerate(utterance["candidates"][-num_candidates:]):
+                        lm_labels = bool(j == num_candidates-1)
+                        instance = build_input_from_segments(persona, history, candidate, tokenizer, lm_labels)
+                        for input_name, input_array in instance.items():
+                            datasets[dataset_name][input_name].append(input_array)
+                    datasets[dataset_name]["mc_labels"].append(num_candidates - 1)
+                    datasets[dataset_name]["n_candidates"] = num_candidates
+                persona = [persona[-1]] + persona[:-1]  # permuted personalities
+
+    logger.info("Pad inputs and convert to Tensor")
+    tensor_datasets = {"train": [], "valid": []}
+    for dataset_name, dataset in datasets.items():
+        dataset = pad_dataset(dataset, padding=tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[-1]))
+        for input_name in MODEL_INPUTS:
+            tensor = torch.tensor(dataset[input_name])
+            if input_name != "mc_labels":
+                tensor = tensor.view((-1, datasets[dataset_name]["n_candidates"]) + tensor.shape[1:])
+            tensor_datasets[dataset_name].append(tensor)
+
+    logger.info("Build train and validation dataloaders")
+    train_dataset, valid_dataset = TensorDataset(*tensor_datasets["train"]), TensorDataset(*tensor_datasets["valid"])
+    train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) if args.distributed else None
+    valid_sampler = torch.utils.data.distributed.DistributedSampler(valid_dataset) if args.distributed else None
+    train_loader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size, shuffle=(not args.distributed))
+    valid_loader = DataLoader(valid_dataset, sampler=valid_sampler, batch_size=args.valid_batch_size, shuffle=False)
+
+    logger.info("Train dataset (Batch, Candidates, Seq length): {}".format(train_dataset.tensors[0].shape))
+    logger.info("Valid dataset (Batch, Candidates, Seq length): {}".format(valid_dataset.tensors[0].shape))
+    return train_loader, valid_loader, train_sampler, valid_sampler
+
+
+def train():
+    parser = ArgumentParser()
+    parser.add_argument("--dataset_path", type=str, default="/Users/tetracycline/repos/datascience/datascience/projects/counsel_chat_all_data_300-tokens.json", help="Path or url of the dataset. If empty download from S3.")
+    parser.add_argument("--dataset_cache", type=str, default='./dataset_cache', help="Path or url of the dataset cache")
+    parser.add_argument("--model_checkpoint", type=str, default="openai-gpt", help="Path, url or short name of the model")
+    parser.add_argument("--num_candidates", type=int, default=2, help="Number of candidates for training")
+    parser.add_argument("--max_history", type=int, default=2, help="Number of previous exchanges to keep in history")
+    parser.add_argument("--train_batch_size", type=int, default=4, help="Batch size for training")
+    parser.add_argument("--valid_batch_size", type=int, default=4, help="Batch size for validation")
+    parser.add_argument("--gradient_accumulation_steps", type=int, default=8, help="Accumulate gradients on several steps")
+    parser.add_argument("--lr", type=float, default=6.25e-5, help="Learning rate")
+    parser.add_argument("--lm_coef", type=float, default=1.0, help="LM loss coefficient")
+    parser.add_argument("--mc_coef", type=float, default=1.0, help="Multiple-choice loss coefficient")
+    parser.add_argument("--max_norm", type=float, default=1.0, help="Clipping gradient norm")
+    parser.add_argument("--n_epochs", type=int, default=3, help="Number of training epochs")
+    parser.add_argument("--personality_permutations", type=int, default=1, help="Number of permutations of personality sentences")
+    parser.add_argument("--eval_before_start", action='store_true', help="If true start with a first evaluation before training")
+    parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", help="Device (cuda or cpu)")
+    parser.add_argument("--fp16", type=str, default="", help="Set to O0, O1, O2 or O3 for fp16 training (see apex documentation)")
+    parser.add_argument("--local_rank", type=int, default=-1, help="Local rank for distributed training (-1: not distributed)")
+    args = parser.parse_args()
+
+    # logging is set to INFO (resp. WARN) for main (resp. auxiliary) process. logger.info => log main process only, logger.warning => log all processes
+    logging.basicConfig(level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
+    logger.warning("Running process %d", args.local_rank)  # This is a logger.warning: it will be printed by all distributed processes
+    logger.info("Arguments: %s", pformat(args))
+
+    # Initialize distributed training if needed
+    args.distributed = (args.local_rank != -1)
+    if args.distributed:
+        torch.cuda.set_device(args.local_rank)
+        args.device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend='nccl', init_method='env://')
+
+    logger.info("Prepare tokenizer, pretrained model and optimizer.")
+    tokenizer_class = GPT2Tokenizer if "gpt2" in args.model_checkpoint else OpenAIGPTTokenizer # cant use Autotokenizer because checkpoint could be a Path
+    tokenizer = tokenizer_class.from_pretrained(args.model_checkpoint)
+
+
+    model_class = GPT2DoubleHeadsModel if "gpt2" in args.model_checkpoint else OpenAIGPTDoubleHeadsModel
+    model = model_class.from_pretrained(args.model_checkpoint)
+    model.to(args.device)
+    # Add special tokens if they are not already added
+    add_special_tokens_(model, tokenizer)
+    optimizer = AdamW(model.parameters(), lr=args.lr, correct_bias=True)
+
+    # Prepare model for FP16 and distributed training if needed (order is important, distributed should be the last)
+    if args.fp16:
+        from apex import amp  # Apex is only required if we use fp16 training
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16)
+    if args.distributed:
+        model = DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank)
+
+    logger.info("Prepare datasets")
+    train_loader, val_loader, train_sampler, valid_sampler = get_data_loaders(args, tokenizer)
+
+    # Training function and trainer
+    def update(engine, batch):
+        model.train()
+        batch = tuple(input_tensor.to(args.device) for input_tensor in batch)
+        input_ids, mc_token_ids, lm_labels, mc_labels, token_type_ids = batch
+        (lm_loss), (mc_loss), *_ = model(
+            input_ids, token_type_ids=token_type_ids, mc_token_ids=mc_token_ids,
+            mc_labels=mc_labels, lm_labels=lm_labels
+        )
+        loss = (lm_loss * args.lm_coef + mc_loss * args.mc_coef) / args.gradient_accumulation_steps
+        if args.fp16:
+            with amp.scale_loss(loss, optimizer) as scaled_loss:
+                scaled_loss.backward()
+            torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_norm)
+        else:
+            loss.backward()
+            torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_norm)
+        if engine.state.iteration % args.gradient_accumulation_steps == 0:
+            optimizer.step()
+            optimizer.zero_grad()
+        return loss.item()
+    trainer = Engine(update)
+
+    # Evaluation function and evaluator (evaluator output is the input of the metrics)
+    def inference(engine, batch):
+        model.eval()
+        with torch.no_grad():
+            batch = tuple(input_tensor.to(args.device) for input_tensor in batch)
+            input_ids, mc_token_ids, lm_labels, mc_labels, token_type_ids = batch
+            logger.info(tokenizer.decode(input_ids[0, -1, :].tolist()))
+            # if we dont send labels to model, it doesnt return losses
+            lm_logits, mc_logits, *_ = model(
+                input_ids, token_type_ids=token_type_ids, mc_token_ids=mc_token_ids,
+            )
+            lm_logits_flat_shifted = lm_logits[..., :-1, :].contiguous().view(-1, lm_logits.size(-1))
+            lm_labels_flat_shifted = lm_labels[..., 1:].contiguous().view(-1)
+            return (lm_logits_flat_shifted, mc_logits), (lm_labels_flat_shifted, mc_labels)
+    evaluator = Engine(inference)
+
+    # Attach evaluation to trainer: we evaluate when we start the training and at the end of each epoch
+    trainer.add_event_handler(Events.EPOCH_COMPLETED, lambda _: evaluator.run(val_loader))
+    if args.n_epochs < 1:
+        trainer.add_event_handler(Events.COMPLETED, lambda _: evaluator.run(val_loader))
+    if args.eval_before_start:
+        trainer.add_event_handler(Events.STARTED, lambda _: evaluator.run(val_loader))
+
+    # Make sure distributed data samplers split the dataset nicely between the distributed processes
+    if args.distributed:
+        trainer.add_event_handler(Events.EPOCH_STARTED, lambda engine: train_sampler.set_epoch(engine.state.epoch))
+        evaluator.add_event_handler(Events.EPOCH_STARTED, lambda engine: valid_sampler.set_epoch(engine.state.epoch))
+
+    # Linearly decrease the learning rate from lr to zero
+    scheduler = PiecewiseLinear(optimizer, "lr", [(0, args.lr), (args.n_epochs * len(train_loader), 0.0)])
+    trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)
+
+    # Prepare metrics - note how we compute distributed metrics
+    RunningAverage(output_transform=lambda x: x).attach(trainer, "loss")
+    metrics = {"nll": Loss(torch.nn.CrossEntropyLoss(ignore_index=-100), output_transform=lambda x: (x[0][0], x[1][0])),
+               "accuracy": Accuracy(output_transform=lambda x: (x[0][1], x[1][1]))}
+    metrics.update({"average_nll": MetricsLambda(average_distributed_scalar, metrics["nll"], args),
+                    "average_accuracy": MetricsLambda(average_distributed_scalar, metrics["accuracy"], args)})
+    metrics["average_ppl"] = MetricsLambda(math.exp, metrics["average_nll"])
+    for name, metric in metrics.items():
+        metric.attach(evaluator, name)
+
+    # On the main process: add progress bar, tensorboard, checkpoints and save model, configuration and tokenizer before we start to train
+    if args.local_rank in [-1, 0]:
+        pbar = ProgressBar(persist=True)
+        pbar.attach(trainer, metric_names=["loss"])
+        evaluator.add_event_handler(Events.COMPLETED, lambda _: pbar.log_message("Validation: %s" % pformat(evaluator.state.metrics)))
+
+        log_dir = make_logdir(args.model_checkpoint)
+        tb_logger = TensorboardLogger(log_dir)
+
+        tb_logger.attach(trainer, log_handler=OutputHandler(tag="training", metric_names=["loss"]), event_name=Events.ITERATION_COMPLETED)
+        tb_logger.attach(trainer, log_handler=OptimizerParamsHandler(optimizer), event_name=Events.ITERATION_STARTED)
+        tb_logger.attach(evaluator, log_handler=OutputHandler(tag="validation", metric_names=list(metrics.keys()), another_engine=trainer), event_name=Events.EPOCH_COMPLETED)
+
+        checkpoint_handler = ModelCheckpoint(log_dir, 'checkpoint', save_interval=1, n_saved=3)
+        trainer.add_event_handler(Events.EPOCH_COMPLETED, checkpoint_handler, {'mymodel': getattr(model, 'module', model)})  # "getattr" takes care of distributed encapsulation
+
+        torch.save(args, log_dir + '/model_training_args.bin')
+        getattr(model, 'module', model).config.to_json_file(os.path.join(log_dir, CONFIG_NAME))
+        tokenizer.save_pretrained(log_dir)
+
+    # Run the training
+    trainer.run(train_loader, max_epochs=args.n_epochs)
+
+    # On the main process: close tensorboard logger and rename the last checkpoint (for easy re-loading with OpenAIGPTModel.from_pretrained method)
+    if args.local_rank in [-1, 0] and args.n_epochs > 0:
+        os.rename(os.path.join(log_dir, checkpoint_handler._saved[-1][1]), os.path.join(log_dir, WEIGHTS_NAME))  # TODO: PR in ignite to have better access to saved file paths (cleaner)
+        tb_logger.close()
+
+if __name__ == "__main__":
+    train()

counsel-chat-master/utils.py

@@ -0,0 +1,387 @@
+from typing import Dict, Any, Callable, List, Tuple, Optional, Union
+import torch
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn import metrics
+from sklearn.utils.multiclass import unique_labels
+from sklearn.base import BaseEstimator, TransformerMixin
+import re
+import logging
+import json
+from datetime import datetime
+import logging
+import os
+import tarfile
+import tempfile
+import socket
+
+import torch
+
+from transformers import cached_path
+
+PERSONACHAT_URL = "https://s3.amazonaws.com/datasets.huggingface.co/personachat/personachat_self_original.json"
+HF_FINETUNED_MODEL = "https://s3.amazonaws.com/models.huggingface.co/transfer-learning-chatbot/gpt_personachat_cache.tar.gz"
+
+logger = logging.getLogger(__file__)
+
+def download_pretrained_model():
+    """ Download and extract finetuned model from S3 """
+    resolved_archive_file = cached_path(HF_FINETUNED_MODEL)
+    tempdir = tempfile.mkdtemp()
+    logger.info("extracting archive file {} to temp dir {}".format(resolved_archive_file, tempdir))
+    with tarfile.open(resolved_archive_file, 'r:gz') as archive:
+        def is_within_directory(directory, target):
+            
+            abs_directory = os.path.abspath(directory)
+            abs_target = os.path.abspath(target)
+        
+            prefix = os.path.commonprefix([abs_directory, abs_target])
+            
+            return prefix == abs_directory
+        
+        def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
+        
+            for member in tar.getmembers():
+                member_path = os.path.join(path, member.name)
+                if not is_within_directory(path, member_path):
+                    raise Exception("Attempted Path Traversal in Tar File")
+        
+            tar.extractall(path, members, numeric_owner=numeric_owner) 
+            
+        
+        safe_extract(archive, tempdir)
+    return tempdir
+
+
+def get_dataset(tokenizer, dataset_path, dataset_cache):
+    """ Get tokenized PERSONACHAT dataset from S3 or cache."""
+    dataset_path = dataset_path or PERSONACHAT_URL
+    dataset_cache = dataset_cache + '_' + type(tokenizer).__name__  # To avoid using GPT cache for GPT-2 and vice-versa
+    if dataset_cache and os.path.isfile(dataset_cache):
+        logger.info("Load tokenized dataset from cache at %s", dataset_cache)
+        dataset = torch.load(dataset_cache)
+    else:
+        logger.info("Download dataset from %s", dataset_path)
+        personachat_file = cached_path(dataset_path)
+        with open(personachat_file, "r", encoding="utf-8") as f:
+            dataset = json.loads(f.read())
+
+        logger.info("Tokenize and encode the dataset")
+        def tokenize(obj):
+            if isinstance(obj, str):
+                return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(obj))
+            if isinstance(obj, dict):
+                return dict((n, tokenize(o)) for n, o in obj.items())
+            return list(tokenize(o) for o in obj)
+        dataset = tokenize(dataset)
+        torch.save(dataset, dataset_cache)
+    return dataset
+
+
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+
+def make_logdir(model_name: str):
+    """Create unique path to save results and checkpoints, e.g. runs/Sep22_19-45-59_gpu-7_gpt2"""
+    # Code copied from ignite repo
+    current_time = datetime.now().strftime('%b%d_%H-%M-%S')
+    logdir = os.path.join(
+        'runs', current_time + '_' + socket.gethostname() + '_' + model_name)
+    return logdir
+
+
+def calculate_classification_metrics(
+    y_true: np.array,
+    y_pred: np.array,
+    average: Optional[str] = None,
+    return_df: bool = True,
+) -> Union[Dict[str, float], pd.DataFrame]:
+    """Computes f1, precision, recall, kappa, accuracy, and support
+
+    Args:
+        y_true: The true labels
+        y_pred: The predicted labels
+        average: How to average multiclass results
+        return_df: Returns a dataframe if true otherwise a dictionary of performance
+            values.
+
+    Returns:
+        Either a dataframe of the performance metrics or a single dictionary
+    """
+    labels = unique_labels(y_true, y_pred)
+
+    # get results
+    precision, recall, f_score, support = metrics.precision_recall_fscore_support(
+        y_true, y_pred, labels=labels, average=average
+    )
+
+    kappa = metrics.cohen_kappa_score(y_true, y_pred, labels=labels)
+    accuracy = metrics.accuracy_score(y_true, y_pred)
+
+    # create a pandas DataFrame
+    if return_df:
+        results = pd.DataFrame(
+            {
+                "class": labels,
+                "f_score": f_score,
+                "precision": precision,
+                "recall": recall,
+                "support": support,
+                "kappa": kappa,
+                "accuracy": accuracy,
+            }
+        )
+    else:
+        results = {
+            "f1": f_score,
+            "precision": precision,
+            "recall": recall,
+            "kappa": kappa,
+            "accuracy": accuracy,
+        }
+
+    return results
+
+
+def visualize_performance(
+        df: pd.DataFrame,
+        metrics: List[str],
+        ax: Optional[Any] = None,
+        title: Optional[str] = None,
+        ylim: Optional[Tuple[float, float]] = None,
+        figsize: Optional[Tuple[int, int]] = None,
+        use_class_names: bool = True
+) -> None:
+    """Takes a Performance DF and converts it to a bar plot performance graph
+
+    Args:
+        df: A dataframe where each row is a class and each column is a metric
+        metrics: A list of metrics from the columns of df to plot
+        ax: A matplotlib axes object that we want to draw the plot on
+        title: The title of the plot
+        ylim: The minimum and maximum range for the yaxis.
+        figsize: The width and height of the figure.  This does nothing if ax is set
+        use_class_names: This will label the x ticks with the class name in a multiclass setting.
+    """
+    unstacked_df = (
+        df[metrics]
+            .T.unstack()
+            .reset_index()
+            .rename(
+            index=str, columns={"level_0": "class", "level_1": "metric", 0: "score"}
+        )
+    )
+
+    if use_class_names:
+        unstacked_df["class"] = unstacked_df["class"].apply(
+            lambda x: df["class"].tolist()[x]
+        )
+
+    if figsize is None:
+        figsize = (10, 7)
+
+    # Diplay the graph
+    if ax is None:
+        fig, ax = plt.subplots(1, 1, figsize=(10, 7))
+
+    sns.barplot(x="class", y="score", hue="metric", data=unstacked_df, ax=ax)
+
+    # Format the graph
+    ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
+    if title is not None:
+        ax.set_title(title, fontsize=20)
+
+    if ylim is not None:
+        ax.set_ylim(ylim)
+
+    plt.tight_layout()
+
+
+class BertTransformer(BaseEstimator, TransformerMixin):
+    """See https://towardsdatascience.com/build-a-bert-sci-kit-transformer-59d60ddd54a5#d608"""
+    def __init__(
+            self,
+            bert_tokenizer,
+            bert_model,
+            max_length: int = 60,
+            embedding_func: Optional[Callable[[Tuple[torch.tensor]], torch.tensor]] = None,
+    ):
+        self.tokenizer = bert_tokenizer
+        self.model = bert_model
+        self.model.eval()
+        self.max_length = max_length
+        self.embedding_func = embedding_func
+
+        if self.embedding_func is None:
+            self.embedding_func = lambda x: x[0][:, 0, :]
+
+        # TODO:: PADDING
+
+    def _tokenize(self, text: str):
+        tokenized_text = self.tokenizer.encode_plus(
+            text, add_special_tokens=True, max_length=self.max_length
+        )["input_ids"]
+        attention_mask = [1] * len(tokenized_text)
+
+        # bert takes in a batch so we need to unsqueeze the rows
+        return (
+            torch.tensor(tokenized_text).unsqueeze(0),
+            torch.tensor(attention_mask).unsqueeze(0),
+        )
+
+    def _tokenize_and_predict(self, text: str):
+        tokenized, attention_mask = self._tokenize(text)
+
+        embeddings = self.model(tokenized, attention_mask)
+        return self.embedding_func(embeddings)
+
+    def transform(self, text: List[str]):
+        if isinstance(text, pd.Series):
+            text = text.tolist()
+
+        with torch.no_grad():
+            return torch.stack([self._tokenize_and_predict(string) for string in text])
+
+    def fit(self, X, y=None):
+        """No fitting necessary so we just return ourselves"""
+        return self
+
+
+def convert_df_to_conv_ai_dict(df: pd.DataFrame,
+                               personality: List[str],
+                               response_columns: List[str],
+                               tokenizer: Callable[[str], List[str]],
+                               max_tokens: Optional[int] = None,
+                               n_candidates: int = 6
+                               ) -> Dict[str, List[Any]]:
+    """
+    Each entry in personachat is a dict with two keys personality and utterances, the dataset is a list of entries.
+    personality:  list of strings containing the personality of the agent
+    utterances: list of dictionaries, each of which has two keys which are lists of strings.
+        candidates: [next_utterance_candidate_1, ..., next_utterance_candidate_19]
+            The last candidate is the ground truth response observed in the conversational data
+        history: [dialog_turn_0, ... dialog_turn N], where N is an odd number since the other user starts every conversation.
+    Preprocessing:
+        - Spaces before periods at end of sentences
+        - everything lowercase
+
+    Process each row of a DataFrame.  For each row:
+    1. Grab the conversational input text
+    2. Grab A the responses
+    3. Create a unique data entry for each response to the question.
+    4. Sample random response sentences from the dataset.
+    5. Combine the random responses into a candidate list.
+
+    Args:
+        df: The counsel chat pandas dataframe
+        personality: The personality we would like to use during training
+        response_columns: Columns which contain valid responses to the question.  For example,
+            the answerText column is the complete response of the therapist
+        tokenizer: The transformers library tokenizer associated with the model we will be
+            training.  It is used for setting the maximum sequence length
+        max_tokens: The maximum number of tokens that any candidate, response, or question should be.
+        n_candidates: The number of candidate phrases to include in the dataset for training.
+            The last member of candidates is the ground truth response
+
+    Returns:
+        A dictionary with a train and validation key.
+    """
+    # Add one because the index of the dataframe is the 0th position.
+    tuple_map = {name: index + 1 for index, name in enumerate(df.columns.tolist())}
+
+    train = []
+    val = []
+    # Step through every row in the dictionary
+    for row in df.itertuples():
+
+        # Get the question name and title
+        # TODO:: MAKE THIS GENERAL YOU DUMB DUMB
+        question_title = row[tuple_map["questionTitle"]]
+        question_text = row[tuple_map["questionText"]]
+        question_combined = question_title + " " + question_text
+
+        # Step through every response column in the row
+        for response_column in response_columns:
+
+            # Get the true response
+            true_response = row[tuple_map[response_column]]
+
+            # We only want to add data if a good response exists
+            if len(true_response) > 1:
+                # Get candidate alternate sentances by sampling from all other questions
+                candidates = sample_candidates(df, row[tuple_map["questionID"]], "questionID", "answerText",
+                                               n_candidates)
+
+                # Add the correct response to the end
+                candidates.append(true_response)
+
+                # We want to trim the size of the tokens
+                if max_tokens is not None:
+                    # Use the provided tokenizer to tokenize the input and truncate at max_tokens
+                    question_combined = tokenizer.convert_tokens_to_string(
+                        tokenizer.tokenize(question_combined)[:max_tokens])
+                    candidates = [tokenizer.convert_tokens_to_string(tokenizer.tokenize(candidate)[:max_tokens]) for
+                                  candidate in candidates]
+
+                if len(candidates) != n_candidates + 1:
+                    print(true_response)
+                    assert False
+
+                # Define the personality and the history
+                d = {"personality": personality,
+                     "utterances": [{"history": [question_combined],
+                                     "candidates": candidates}]}
+                if getattr(row, "split") == "train":
+                    train.append(d)
+                elif getattr(row, "split") == "val":
+                    val.append(d)
+
+    data = {"train": train, "valid": val}
+
+    return data
+
+
+def sample_candidates(df: pd.DataFrame, current_id: Any, id_column: str, text_column: str, n: int) -> List[str]:
+    """Samples candidate responses to a question from the dataframe
+
+    It is aware of data splits and only samples from within the same split.  This avoids
+    leaking information between training validation and testing.  The sampled responses are
+    also drawn from all rows which do not have the same id as the current_id
+
+    Args:
+        df: The dataframe we want to sample responses from
+        current_id: The unique identifier we would like to leave out of our sampling
+        id_column: The column name in the dataframe with the unique ids.  current_id should
+            be an element of this column
+        text_column: The column with the text we want to sample
+        n: How many samples we want to take.
+
+    Returns:
+        A list of samples strings from our dataframe.
+    """
+    # We must only sample candidates from the correct data split to avoid information leakage across channels
+    split = df[df[id_column] == current_id]["split"].tolist()[0]
+    candidate_df = df[df["split"] == split]
+
+    # Sample 3 random rows from the dataframe not matching the current id
+    sampled_texts = candidate_df[candidate_df[id_column] != current_id].sample(n + 15)[text_column].tolist()
+
+    # join them all
+    text = " ".join(sampled_texts)
+
+    # Replace all newlines with spaces...
+    text_no_newline = re.sub("\n", " ", text).lower()
+
+    # Split on punctuation
+    split_text = re.split('[?.!]', text_no_newline)
+
+    # Remove all empty lines
+    filtered_text = [x.strip() for x in split_text if len(x.strip()) > 1]
+
+    # Shuffle the list
+    return np.random.choice(filtered_text, n).tolist()

counsel-chat-master/utils_test.py

@@ -0,0 +1,50 @@
+from typing imprt List
+import unittest
+
+import ddt
+from ddt import unpack, data
+import numpy as np
+import pandas as pd
+from transformers import OpenAIGPTTokenizer
+
+from utils import sample_candidates, convert_df_to_conv_ai_dict
+
+
+# fmt: off
+class UtilsTest(unittest.TestCase):
+
+    def test_sample_candidates(self):
+        # Test that split id doesn't show up
+        # Create random DF
+        df = pd.DataFrame(data, columns=["questionID", "answerText", 'split'])
+
+        for i in range(5):
+            candidates = sample_candidates(df, , "questionID", "answerText", n_candidates)
+            # Check that the samples don't come from the true data
+
+    def test_fuzz_convert_df_to_conv_ai_dict(self):
+        df = pd.read_csv("data/20200325_counsel_chat.csv")
+        df = df[df["split"] == "train"]
+        tokenizer = OpenAIGPTTokenizer.from_pretrained("openai-gpt")
+        for i in range(5):
+            temp_df = df.sample(100)
+            max_tokens = np.random.randint(1, 200)
+            n_candidates = np.random.randint(1, 10)
+            d = convert_df_to_conv_ai_dict(temp_df,
+                                           [""],
+                                           ["answerText"],
+                                           tokenizer,
+                                           max_tokens=max_tokens,
+                                           n_candidates=n_candidates)
+
+            # Test max length
+            self.assertLessEqual(max([len(x["utterances"][0]["history"][0].split()) for x in d["train"]]), max_tokens)
+
+            # Test n_candidates is equal to the number in the candidates list plus the one true response.
+            train_lengths = [len(x["utterances"][0]["candidates"]) for x in d["train"]]
+            self.assertEqual(n_candidates + 1, max(train_lengths))
+            self.assertEqual(n_candidates + 1, min(train_lengths))
+
+
+if __name__ == "__main__":
+    unittest.main()

counselchat-convai/added_tokens.json

@@ -0,0 +1 @@
+{"<bos>": 40478, "<eos>": 40479, "<pad>": 40480, "<speaker1>": 40481, "<speaker2>": 40482}

counselchat-convai/config.json

@@ -0,0 +1,54 @@
+{
+  "afn": "gelu",
+  "architectures": [
+    "OpenAIGPTLMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": null,
+  "do_sample": false,
+  "embd_pdrop": 0.1,
+  "eos_token_ids": null,
+  "finetuning_task": null,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "initializer_range": 0.02,
+  "is_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "layer_norm_epsilon": 1e-05,
+  "length_penalty": 1.0,
+  "max_length": 20,
+  "model_type": "openai-gpt",
+  "n_ctx": 512,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_layer": 12,
+  "n_positions": 512,
+  "n_special": 0,
+  "num_beams": 1,
+  "num_labels": 1,
+  "num_return_sequences": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "output_past": true,
+  "pad_token_id": null,
+  "predict_special_tokens": true,
+  "pruned_heads": {},
+  "repetition_penalty": 1.0,
+  "resid_pdrop": 0.1,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "temperature": 1.0,
+  "top_k": 50,
+  "top_p": 1.0,
+  "torchscript": false,
+  "use_bfloat16": false,
+  "vocab_size": 40483
+}

counselchat-convai/events.out.tfevents.1586138049.3b21657f0ca9

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2c720c161121fb6a7457b091cdfb68090b7a55ab9dc7090a4d93a9791077c5b
+size 140048

counselchat-convai/model_training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65010fa713a71eac503cd4522be309b75f2874212050c38a780c5c7d7b73dfa9
+size 729

counselchat-convai/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd9ab879baa4977df5caba66f5b923729f5618aee99b7aa3f5fe244a9e59150d
+size 478771216

counselchat-convai/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<bos>", "eos_token": "<eos>", "unk_token": "<unk>", "pad_token": "<pad>", "additional_special_tokens": ["<speaker1>", "<speaker2>"]}

counselchat-convai/tokenizer_config.json

@@ -0,0 +1 @@
+{"max_len": 512}