| { | |
| "monitor_metric": "eval_exact_match", | |
| "trainer_kwargs": { | |
| "class_name": "RA_Reader" | |
| }, | |
| "trainee_kwargs": { | |
| "class_name": "Encoder_Generator", | |
| "loss": { | |
| "class_name": "NLLLoss" | |
| }, | |
| "freeze_prefixes": [], | |
| "use_image": true, | |
| "quantize": false, | |
| "codebook_kwargs": { | |
| "class_name": "GroupedResidualVQ", | |
| "decay": 0.5, | |
| "commitment_weight": 1, | |
| "dim": 1280, | |
| "codebook_size": 1024, | |
| "num_quantizers": 128, | |
| "groups": 8, | |
| "sample_codebook_temp": 0.0, | |
| "learnable_codebook": false, | |
| "use_cosine_sim": false, | |
| "shared_codebook": false, | |
| "kmeans_init": false | |
| } | |
| }, | |
| "data_module_kwargs": { | |
| "class_name": "multi_modal_DataModule", | |
| "data_processor": { | |
| "class_name": "viquae_data_processor", | |
| "dataset_path": "../../all_data/", | |
| "kb_path": "../../all_data/passages/", | |
| "entity_kb_path": "../../all_data/kb/" | |
| }, | |
| "relevant_indices_key": "BM25_provenance_indices", | |
| "irrelevant_indices_key": "BM25_irrelevant_indices", | |
| "input_key": "input", | |
| "passage_key": "passage", | |
| "dataloader_kwargs": { | |
| "num_workers": 6, | |
| "prefetch_factor": 2 | |
| }, | |
| "image_processor_kwargs": { | |
| "class_name": "ImageFormatter", | |
| "feature_extractor_kwargs": { | |
| "class_name": "CLIPFeatureExtractor", | |
| "pretrained_model_name_or_path": "clip-vit-base-patch32_FE" | |
| } | |
| } | |
| }, | |
| "answer_generator_kwargs": { | |
| "class_name": "Blip2ForConditionalGeneration", | |
| "checkpoint_name": "answer_generator", | |
| "checkpoint": "saved_models/blip2_triviaqa/answer_generator/pytorch_model.bin", | |
| "title_key": "wikipedia_title", | |
| "column": "my_DPR_few_shot", | |
| "pretrained_model_name_or_path": "/home/data/meerqat/my_transformers_cache/blip2", | |
| "k_train": 5, | |
| "k_test": 5, | |
| "k_entities": 3, | |
| "rag_training": false, | |
| "rag": false, | |
| "generator_only": true, | |
| "entity_prompt": false, | |
| "beam_search": true, | |
| "best_answer": false, | |
| "best_entity": false, | |
| "tokenizer_kwargs": { | |
| "class_name": "Blip2Processor", | |
| "pretrained_model_name_or_path": "blip2_processor" | |
| }, | |
| "image_processor_kwargs": { | |
| "class_name": "ImageFormatter", | |
| "feature_extractor_kwargs": { | |
| "class_name": "Blip2Processor", | |
| "pretrained_model_name_or_path": "blip2_processor" | |
| } | |
| }, | |
| "inference_path": "saved_models/blip2_triviaqa_viquae/answer_generator" | |
| } | |
| } | |