mistralai

/

Mixtral-8x7B-Instruct-v0.1

like 4.12k

model = AutoModelForCausalLM.from_pretrained(
model_id,
device_map='auto',
quantization_config=nf4_config,
use_cache=True,
attn_implementation="flash_attention_2"
​
)

ImportError Traceback (most recent call last)
/opt/conda/lib/python3.8/site-packages/transformers/utils/import_utils.py in _get_module(self, module_name)
1381 setattr(self, name, value)
-> 1382 return value
1383

/opt/conda/lib/python3.8/importlib/init.py in import_module(name, package)
126 level += 1
--> 127 return _bootstrap._gcd_import(name[level:], package, level)
128

/opt/conda/lib/python3.8/importlib/_bootstrap.py in _gcd_import(name, package, level)

/opt/conda/lib/python3.8/importlib/_bootstrap.py in find_and_load(name, import)

/opt/conda/lib/python3.8/importlib/_bootstrap.py in find_and_load_unlocked(name, import)

/opt/conda/lib/python3.8/importlib/_bootstrap.py in _load_unlocked(spec)

/opt/conda/lib/python3.8/importlib/_bootstrap_external.py in exec_module(self, module)

/opt/conda/lib/python3.8/importlib/_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

/opt/conda/lib/python3.8/site-packages/transformers/models/mixtral/modeling_mixtral.py in
57 if is_flash_attn_2_available():
---> 58 from flash_attn import flash_attn_func, flash_attn_varlen_func
59 from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa

/opt/conda/lib/python3.8/site-packages/flash_attn/init.py in
2
----> 3 from flash_attn.flash_attn_interface import (
4 flash_attn_func,

/opt/conda/lib/python3.8/site-packages/flash_attn/flash_attn_interface.py in
9 # We need to import the CUDA kernels after importing torch
---> 10 import flash_attn_2_cuda as flash_attn_cuda
11

ImportError: libcudart.so.12: cannot open shared object file: No such file or directory

The above exception was the direct cause of the following exception:

RuntimeError Traceback (most recent call last)
in
----> 1 model = AutoModelForCausalLM.from_pretrained(
2 model_id,
3 device_map='auto',
4 quantization_config=nf4_config,
5 use_cache=True,

/opt/conda/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
563 )
564 elif type(config) in cls._model_mapping.keys():
--> 565 model_class = _get_model_class(config, cls._model_mapping)
566 return model_class.from_pretrained(
567 pretrained_model_name_or_path, *model_args, config=config, **hub_kwargs, **kwargs

/opt/conda/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py in _get_model_class(config, model_mapping)
385
386 def _get_model_class(config, model_mapping):
--> 387 supported_models = model_mapping[type(config)]
388 if not isinstance(supported_models, (list, tuple)):
389 return supported_models

/opt/conda/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py in getitem(self, key)
738 if model_type in self._model_mapping:
739 model_name = self._model_mapping[model_type]
--> 740 return self._load_attr_from_module(model_type, model_name)
741
742 # Maybe there was several model types associated with this config.

/opt/conda/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py in _load_attr_from_module(self, model_type, attr)
752 if module_name not in self._modules:
753 self._modules[module_name] = importlib.import_module(f".{module_name}", "transformers.models")
--> 754 return getattribute_from_module(self._modules[module_name], attr)
755
756 def keys(self):

/opt/conda/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py in getattribute_from_module(module, attr)
696 if isinstance(attr, tuple):
697 return tuple(getattribute_from_module(module, a) for a in attr)
--> 698 if hasattr(module, attr):
699 return getattr(module, attr)
700 # Some of the mappings have entries model_type -> object of another model type. In that case we try to grab the

/opt/conda/lib/python3.8/site-packages/transformers/utils/import_utils.py in getattr(self, name)
1370 def getattr(self, name: str) -> Any:
1371 if name in self._objects:
-> 1372 return self._objects[name]
1373 if name in self._modules:
1374 value = self._get_module(name)

/opt/conda/lib/python3.8/site-packages/transformers/utils/import_utils.py in _get_module(self, module_name)
1382 return value
1383
-> 1384 def _get_module(self, module_name: str):
1385 try:
1386 return importlib.import_module("." + module_name, self.name)

RuntimeError: Failed to import transformers.models.mixtral.modeling_mixtral because of the following error (look up to see its traceback):
libcudart.so.12: cannot open shared object file: No such file or directory