|
|
|
|
|
"""PyTorch LLaMA model.""" |
|
|
|
import json |
|
from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union |
|
|
|
import torch |
|
import torch.utils.checkpoint |
|
|
|
from transformers.generation.configuration_utils import GenerationConfig |
|
from transformers.generation.logits_process import LogitsProcessorList |
|
from transformers.generation.stopping_criteria import StoppingCriteriaList |
|
from transformers.generation.utils import ( |
|
GenerateBeamDecoderOnlyOutput, |
|
GenerateBeamEncoderDecoderOutput, |
|
GenerateDecoderOnlyOutput, |
|
GenerateEncoderDecoderOutput |
|
) |
|
from transformers.models.llama.modeling_llama import LlamaForCausalLM |
|
from transformers.utils import logging |
|
|
|
|
|
if TYPE_CHECKING: |
|
from transformers.modeling_utils import PreTrainedModel |
|
from transformers.generation.streamers import BaseStreamer |
|
|
|
logger = logging.get_logger(__name__) |
|
|
|
GenerateNonBeamOutput = Union[GenerateDecoderOnlyOutput, GenerateEncoderDecoderOutput] |
|
GenerateBeamOutput = Union[GenerateBeamDecoderOnlyOutput, GenerateBeamEncoderDecoderOutput] |
|
GenerateOutput = Union[GenerateNonBeamOutput, GenerateBeamOutput] |
|
|
|
|
|
class FunctionaryForCausalLM(LlamaForCausalLM): |
|
|
|
def generate_tool_use( |
|
self, |
|
inputs: Optional[torch.Tensor] = None, |
|
generation_config: Optional[GenerationConfig] = None, |
|
logits_processor: Optional[LogitsProcessorList] = None, |
|
stopping_criteria: Optional[StoppingCriteriaList] = None, |
|
prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None, |
|
synced_gpus: Optional[bool] = None, |
|
assistant_model: Optional["PreTrainedModel"] = None, |
|
streamer: Optional["BaseStreamer"] = None, |
|
negative_prompt_ids: Optional[torch.Tensor] = None, |
|
negative_prompt_attention_mask: Optional[torch.Tensor] = None, |
|
**kwargs, |
|
) -> Union[GenerateOutput, torch.LongTensor]: |
|
|
|
tokenizer = kwargs.pop("tokenizer", None) |
|
|
|
results = self.generate( |
|
inputs=inputs, |
|
generation_config=generation_config, |
|
logits_processor=logits_processor, |
|
stopping_criteria=stopping_criteria, |
|
prefix_allowed_tokens_fn=prefix_allowed_tokens_fn, |
|
synced_gpus=synced_gpus, |
|
assistant_model=assistant_model, |
|
streamer=streamer, |
|
negative_prompt_ids=negative_prompt_ids, |
|
negative_prompt_attention_mask=negative_prompt_attention_mask, |
|
**kwargs, |
|
) |
|
|
|
input_ids = kwargs.pop("input_ids") |
|
function_call_token = ">>>" |
|
|
|
correct_results = [] |
|
for input_id, result in zip(input_ids, results): |
|
final_output_json = {"role": "assistant", "content": None, "tool_calls": None} |
|
tool_calls = [] |
|
raw_output_str = tokenizer.decode(result[len(input_id):].cpu()) |
|
chunks = raw_output_str.split(function_call_token) |
|
for i, chunk in enumerate(chunks): |
|
if len(chunk) == 0: |
|
continue |
|
|
|
chunk = chunk.replace(tokenizer.pad_token, "") |
|
has_text = True if chunk.startswith("all") else False |
|
if i == 0 and has_text is not False: |
|
final_output_json["content"] = chunk.strip[:-len("<|eot_id|>")] if chunk.endswith("<|eot_id|>") else chunk |
|
final_output_json["content"] = final_output_json["content"][len("all\n"):] |
|
else: |
|
tool_calls.append( |
|
{ |
|
"name": chunk[: chunk.index("\n{")], |
|
"arguments": chunk[chunk.index("\n{") + 1: -len("<|eot_id|>")] if chunk.endswith("<|eot_id|>") else chunk[chunk.index("\n{") + 1:] |
|
} |
|
) |
|
if len(tool_calls) > 0: |
|
final_output_json["tool_calls"] = tool_calls |
|
final_output_str = json.dumps(final_output_json, indent=4) |
|
final_output_ids = tokenizer(final_output_str, add_special_tokens=False)["input_ids"] |
|
correct_results.append( |
|
torch.cat( |
|
(result[:len(input_id)].cpu(), torch.tensor(final_output_ids)) |
|
) |
|
) |
|
max_len = max([tensor.shape[0] for tensor in correct_results]) |
|
correct_results = [ |
|
torch.nn.functional.pad( |
|
correct_result, (0, max_len - correct_result.shape[0]), value=tokenizer.eos_token_id |
|
) for correct_result in correct_results |
|
] |
|
correct_results = torch.stack(correct_results) |
|
|
|
return correct_results |
