upload (#1)

- initial upload (232893bd8f2f89de25ac9dc4df0b62bb9d4a4404)
- Update custom_interface.py (42c29e0a9b45031ff4171bc947cc205e5898f00f)


Co-authored-by: Yingzhi Wang <[email protected]>

.gitattributes

@@ -30,3 +30,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model.ckpt filter=lfs diff=lfs merge=lfs -text
+wav2vec2.ckpt filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,116 @@
 ---
-license: apache-2.0
+language: zh
+thumbnail:
+tags:
+- automatic-speech-recognition
+- CTC
+- wav2vec2
+- pytorch
+- speechbrain
+license: "apache-2.0"
+datasets:
+- aishell
+metrics:
+- cer
 ---
+
+<iframe src="https://ghbtns.com/github-btn.html?user=speechbrain&repo=speechbrain&type=star&count=true&size=large&v=2" frameborder="0" scrolling="0" width="170" height="30" title="GitHub"></iframe>
+<br/><br/>
+
+# Wav2vec2-CTC for AISHELL (Mandarin Chinese)
+
+This repository provides all the necessary tools to perform automatic speech
+recognition from an end-to-end system pretrained on AISHELL (Mandarin Chinese)
+within SpeechBrain. For a better experience, we encourage you to learn more about
+[SpeechBrain](https://speechbrain.github.io).
+
+The performance of the model is the following:
+
+| Release | Dev CER | Test CER | GPUs | Full Results |
+|:-------------:|:--------------:|:--------------:|:--------:|:--------:|
+| 20-09-22 | 4.48 | 5.02 | 1xRTX8000 48GB | [Google Drive](https://drive.google.com/drive/folders/1GTB5IzQPl57j-0I1IpmvKg722Ti4ahLz?usp=sharing)|
+
+
+
+## Pipeline description
+
+This ASR system is composed of 2 different but linked blocks:
+- Tokenizer (from huggingface) that transforms words into chars and trained with
+the training transcriptions of AISHELL-1.
+- Acoustic model made of a wav2vec2 encoder and fully-connected layers
+
+To Train this system from scratch, [see our SpeechBrain recipe](https://github.com/speechbrain/speechbrain/tree/develop/recipes/AISHELL-1/ASR/CTC).
+
+The system is trained with recordings sampled at 16kHz (single channel).
+
+## Install SpeechBrain
+
+First of all, please install SpeechBrain with the following command:
+
+```
+pip install speechbrain
+```
+
+Please notice that we encourage you to read our tutorials and learn more about
+[SpeechBrain](https://speechbrain.github.io).
+
+### Transcribing your own audio files (in English)
+
+```python
+from speechbrain.pretrained.interfaces import foreign_class
+asr_model = foreign_class(source="speechbrain/asr-wav2vec2-ctc-aishell",  pymodule_file="custom_interface.py", classname="CustomEncoderDecoderASR")
+asr_model.transcribe_file("speechbrain/asr-wav2vec2-ctc-aishell/example.wav")
+```
+
+### Inference on GPU
+To perform inference on the GPU, add  `run_opts={"device":"cuda"}`  when calling the `from_hparams` method.
+
+## Parallel Inference on a Batch
+Please, [see this Colab notebook](https://colab.research.google.com/drive/1hX5ZI9S4jHIjahFCZnhwwQmFoGAi3tmu?usp=sharing) to figure out how to transcribe in parallel a batch of input sentences using a pre-trained model.
+
+### Training
+The model was trained with SpeechBrain (Commit hash: '480dde87').
+To train it from scratch follow these steps:
+1. Clone SpeechBrain:
+```bash
+git clone https://github.com/speechbrain/speechbrain/git
+```
+2. Install it:
+```bash
+cd speechbrain
+pip install -r requirements.txt
+pip install -e .
+```
+
+3. Run Training:
+```bash
+cd recipes/AISHELL-1/ASR/CTC/
+python train_with_wav2vec.py hparams/train_with_wav2vec.yaml --data_folder=your_data_folder
+```
+
+You can find our training results (models, logs, etc) [here](https://drive.google.com/drive/folders/1GTB5IzQPl57j-0I1IpmvKg722Ti4ahLz?usp=sharing).
+
+### Limitations
+The SpeechBrain team does not provide any warranty on the performance achieved by this model when used on other datasets.
+
+
+# **About SpeechBrain**
+- Website: https://speechbrain.github.io/
+- Code: https://github.com/speechbrain/speechbrain/
+- HuggingFace: https://huggingface.co/speechbrain/
+
+
+# **Citing SpeechBrain**
+Please, cite SpeechBrain if you use it for your research or business.
+
+```bibtex
+@misc{speechbrain,
+  title={{SpeechBrain}: A General-Purpose Speech Toolkit},
+  author={Mirco Ravanelli and Titouan Parcollet and Peter Plantinga and Aku Rouhe and Samuele Cornell and Loren Lugosch and Cem Subakan and Nauman Dawalatabad and Abdelwahab Heba and Jianyuan Zhong and Ju-Chieh Chou and Sung-Lin Yeh and Szu-Wei Fu and Chien-Feng Liao and Elena Rastorgueva and François Grondin and William Aris and Hwidong Na and Yan Gao and Renato De Mori and Yoshua Bengio},
+  year={2021},
+  eprint={2106.04624},
+  archivePrefix={arXiv},
+  primaryClass={eess.AS},
+  note={arXiv:2106.04624}
+}
+```

custom_interface.py

@@ -0,0 +1,130 @@
+"""Custom Interface for AISHELL-1 CTC inference
+An external tokenizer is used so some special tokens
+need to be specified during decoding
+
+Authors
+ * Yingzhi Wang 2022
+"""
+
+import torch
+from speechbrain.pretrained import Pretrained
+
+
+class CustomEncoderDecoderASR(Pretrained):
+    """A ready-to-use Encoder-Decoder ASR model
+    The class can be used either to run only the encoder (encode()) to extract
+    features or to run the entire encoder-decoder model
+    (transcribe()) to transcribe speech. The given YAML must contains the fields
+    specified in the *_NEEDED[] lists.
+    Example
+    -------
+    >>> from speechbrain.pretrained import EncoderDecoderASR
+    >>> tmpdir = getfixture("tmpdir")
+    >>> asr_model = EncoderDecoderASR.from_hparams(
+    ...     source="speechbrain/asr-crdnn-rnnlm-librispeech",
+    ...     savedir=tmpdir,
+    ... )
+    >>> asr_model.transcribe_file("tests/samples/single-mic/example2.flac")
+    "MY FATHER HAS REVEALED THE CULPRIT'S NAME"
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.tokenizer = self.hparams.tokenizer
+
+    def transcribe_file(self, path):
+        """Transcribes the given audiofile into a sequence of words.
+        Arguments
+        ---------
+        path : str
+            Path to audio file which to transcribe.
+        Returns
+        -------
+        str
+            The audiofile transcription produced by this ASR system.
+        """
+        waveform = self.load_audio(path)
+        # Fake a batch:
+        batch = waveform.unsqueeze(0)
+        rel_length = torch.tensor([1.0])
+        predicted_words = self.transcribe_batch(
+            batch, rel_length
+        )
+        return predicted_words[0]
+
+    def encode_batch(self, wavs):
+        """Encodes the input audio into a sequence of hidden states
+        The waveforms should already be in the model's desired format.
+        You can call:
+        ``normalized = EncoderDecoderASR.normalizer(signal, sample_rate)``
+        to get a correctly converted signal in most cases.
+        Arguments
+        ---------
+        wavs : torch.tensor
+            Batch of waveforms [batch, time, channels] or [batch, time]
+            depending on the model.
+        wav_lens : torch.tensor
+            Lengths of the waveforms relative to the longest one in the
+            batch, tensor of shape [batch]. The longest one should have
+            relative length 1.0 and others len(waveform) / max_length.
+            Used for ignoring padding.
+        Returns
+        -------
+        torch.tensor
+            The encoded batch
+        """
+        wavs = wavs.float()
+        wavs = wavs.to(self.device)
+        outputs = self.mods.wav2vec2(wavs)
+        outputs = self.mods.enc(outputs)
+        outputs = self.mods.ctc_lin(outputs)
+        return outputs
+
+    def transcribe_batch(self, wavs, wav_lens):
+        """Transcribes the input audio into a sequence of words
+        The waveforms should already be in the model's desired format.
+        You can call:
+        ``normalized = EncoderDecoderASR.normalizer(signal, sample_rate)``
+        to get a correctly converted signal in most cases.
+        Arguments
+        ---------
+        wavs : torch.tensor
+            Batch of waveforms [batch, time, channels] or [batch, time]
+            depending on the model.
+        wav_lens : torch.tensor
+            Lengths of the waveforms relative to the longest one in the
+            batch, tensor of shape [batch]. The longest one should have
+            relative length 1.0 and others len(waveform) / max_length.
+            Used for ignoring padding.
+        Returns
+        -------
+        list
+            Each waveform in the batch transcribed.
+        tensor
+            Each predicted token id.
+        """
+        with torch.no_grad():
+            wav_lens = wav_lens.to(self.device)
+            encoder_out = self.encode_batch(wavs)
+            p_ctc = self.hparams.log_softmax(encoder_out)
+            sequences = self.hparams.decoder(p_ctc, wav_lens)
+            predicted_words_list = []
+            for sequence in sequences:
+                predicted_tokens = self.tokenizer.convert_ids_to_tokens(
+                    sequence
+                )
+                predicted_words = []
+                for c in predicted_tokens:
+                    if c == "[CLS]":
+                        continue
+                    elif c == "[SEP]" or c == "[PAD]":
+                        break
+                    else:
+                        predicted_words.append(c)
+                predicted_words_list.append(predicted_words)
+
+        return predicted_words_list
+
+    def forward(self, wavs, wav_lens):
+        """Runs full transcription - note: no gradients through decoding"""
+        return self.transcribe_batch(wavs, wav_lens)

hyperparams.yaml

@@ -0,0 +1,78 @@
+# ############################################################################
+# Model: CTC-wav2vec2
+# Encoder: wav2vec2
+# Decoder: -
+# Tokens: Char
+# losses: CTC
+# Training: AISHELL-1
+# Authors:  Yingzhi WANG 2022
+# ############################################################################
+
+wav2vec2_hub: TencentGameMate/chinese-wav2vec2-large
+sample_rate: 16000
+
+wav2vec_output_dim: 1024
+dnn_neurons: 1024
+freeze_wav2vec: False
+dropout: 0.15
+
+tokenizer: !apply:transformers.BertTokenizer.from_pretrained
+   pretrained_model_name_or_path: bert-base-chinese
+
+output_neurons: 21128
+
+# Decoding parameters
+# Be sure that the bos and eos index match with the BPEs ones
+blank_index: 0
+
+enc: !new:speechbrain.nnet.containers.Sequential
+   input_shape: [null, null, !ref <wav2vec_output_dim>]
+   linear1: !name:speechbrain.nnet.linear.Linear
+      n_neurons: !ref <dnn_neurons>
+      bias: True
+   bn1: !name:speechbrain.nnet.normalization.BatchNorm1d
+   activation: !new:torch.nn.LeakyReLU
+   drop: !new:torch.nn.Dropout
+      p: !ref <dropout>
+   linear2: !name:speechbrain.nnet.linear.Linear
+      n_neurons: !ref <dnn_neurons>
+      bias: True
+   bn2: !name:speechbrain.nnet.normalization.BatchNorm1d
+   activation2: !new:torch.nn.LeakyReLU
+   drop2: !new:torch.nn.Dropout
+      p: !ref <dropout>
+   linear3: !name:speechbrain.nnet.linear.Linear
+      n_neurons: !ref <dnn_neurons>
+      bias: True
+   bn3: !name:speechbrain.nnet.normalization.BatchNorm1d
+   activation3: !new:torch.nn.LeakyReLU
+
+wav2vec2: !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
+   source: !ref <wav2vec2_hub>
+   output_norm: True
+   freeze: !ref <freeze_wav2vec>
+   save_path: model_checkpoints
+
+ctc_lin: !new:speechbrain.nnet.linear.Linear
+   input_size: !ref <dnn_neurons>
+   n_neurons: !ref <output_neurons>
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+   apply_log: True
+
+decoder: !name:speechbrain.decoders.ctc_greedy_decode
+   blank_id: !ref <blank_index>
+
+model: !new:torch.nn.ModuleList
+   - [!ref <enc>, !ref <ctc_lin>]
+
+modules:
+   wav2vec2: !ref <wav2vec2>
+   enc: !ref <enc>
+   ctc_lin: !ref <ctc_lin>
+
+pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
+   loadables:
+      wav2vec2: !ref <wav2vec2>
+      model: !ref <model>
+

model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d793d99305f11da0a3c058e03e1fd370b5d93de512c008c0709e8b91dee608ea
+size 99276064

wav2vec2.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1e4c8a9b2ab99c2c20d7b0d9c970615033faf51794cfce3e59edc214c92df65
+size 1261923125