Update README with Transformers usage

README.md

@@ -16,6 +16,7 @@ datasets:
 - multilingual_librispeech
 - mozilla-foundation/common_voice_8_0
 - MLCommons/peoples_speech
+thumbnail: null
 tags:
 - automatic-speech-recognition
 - speech
@@ -26,21 +27,19 @@ tags:
 - pytorch
 - NeMo
 - hf-asr-leaderboard
+- transformers
 license: cc-by-4.0
 widget:
 - example_title: Librispeech sample 1
   src: https://cdn-media.huggingface.co/speech_samples/sample1.flac
 - example_title: Librispeech sample 2
   src: https://cdn-media.huggingface.co/speech_samples/sample2.flac
-metrics:
-- wer
-pipeline_tag: automatic-speech-recognition
 model-index:
 - name: parakeet_rnnt_1.1b
   results:
   - task:
-      type: automatic-speech-recognition
       name: Automatic Speech Recognition
+      type: automatic-speech-recognition
     dataset:
       name: AMI (Meetings test)
       type: edinburghcstr/ami
@@ -49,12 +48,12 @@ model-index:
       args:
         language: en
     metrics:
-    - type: wer
-      value: 17.1
-      name: Test WER
+    - name: Test WER
+      type: wer
+      value: 17.10
   - task:
-      type: automatic-speech-recognition
       name: Automatic Speech Recognition
+      type: automatic-speech-recognition
     dataset:
       name: Earnings-22
       type: revdotcom/earnings22
@@ -62,12 +61,12 @@ model-index:
       args:
         language: en
     metrics:
-    - type: wer
+    - name: Test WER
+      type: wer
       value: 14.11
-      name: Test WER
   - task:
-      type: automatic-speech-recognition
       name: Automatic Speech Recognition
+      type: automatic-speech-recognition
     dataset:
       name: GigaSpeech
       type: speechcolab/gigaspeech
@@ -75,12 +74,12 @@ model-index:
       args:
         language: en
     metrics:
-    - type: wer
+    - name: Test WER
+      type: wer
       value: 9.96
-      name: Test WER
   - task:
-      type: automatic-speech-recognition
       name: Automatic Speech Recognition
+      type: automatic-speech-recognition
     dataset:
       name: LibriSpeech (clean)
       type: librispeech_asr
@@ -89,12 +88,23 @@ model-index:
       args:
         language: en
     metrics:
-    - type: wer
+    - name: Test WER
+      type: wer
       value: 1.46
-      name: Test WER
-    - type: wer
+  - task:
+      name: Automatic Speech Recognition
+      type: automatic-speech-recognition
+    dataset:
+      name: LibriSpeech (other)
+      type: librispeech_asr
+      config: other
+      split: test
+      args:
+        language: en
+    metrics:
+    - name: Test WER
+      type: wer
       value: 2.47
-      name: Test WER
   - task:
       type: Automatic Speech Recognition
       name: automatic-speech-recognition
@@ -106,9 +116,9 @@ model-index:
       args:
         language: en
     metrics:
-    - type: wer
+    - name: Test WER
+      type: wer
       value: 3.11
-      name: Test WER
   - task:
       type: Automatic Speech Recognition
       name: automatic-speech-recognition
@@ -120,12 +130,12 @@ model-index:
       args:
         language: en
     metrics:
-    - type: wer
+    - name: Test WER
+      type: wer
       value: 3.92
-      name: Test WER
   - task:
-      type: automatic-speech-recognition
       name: Automatic Speech Recognition
+      type: automatic-speech-recognition
     dataset:
       name: Vox Populi
       type: facebook/voxpopuli
@@ -134,9 +144,9 @@ model-index:
       args:
         language: en
     metrics:
-    - type: wer
+    - name: Test WER
+      type: wer
       value: 5.39
-      name: Test WER
   - task:
       type: Automatic Speech Recognition
       name: automatic-speech-recognition
@@ -148,9 +158,13 @@ model-index:
       args:
         language: en
     metrics:
-    - type: wer
+    - name: Test WER
+      type: wer
       value: 5.79
-      name: Test WER
+  
+metrics:
+- wer
+pipeline_tag: automatic-speech-recognition
 ---
 
 # Parakeet RNNT 1.1B (en)
@@ -184,25 +198,27 @@ NVIDIA Developer [Nemotron](https://developer.nvidia.com/nemotron)<br>
 [NVIDIA Riva Speech](https://developer.nvidia.com/riva?sortBy=developer_learning_library%2Fsort%2Ffeatured_in.riva%3Adesc%2Ctitle%3Aasc#demos)<br>
 [NeMo Documentation](https://docs.nvidia.com/nemo-framework/user-guide/latest/nemotoolkit/asr/models.html)<br>
 
-## NVIDIA NeMo: Training
+## How to Use this Model
+
+The model is available for use in the NeMo toolkit [3], and can be used as a pre-trained checkpoint for inference or for fine-tuning on another dataset.
+
+You can also run Parakeet RNNT with [Transformers](https://github.com/huggingface/transformers) 🤗 (more below).
+
+### 1) NeMo usage
 
 To train, fine-tune or play with the model you will need to install [NVIDIA NeMo](https://github.com/NVIDIA/NeMo). We recommend you install it after you've installed latest PyTorch version.
 ```
 pip install nemo_toolkit['all']
 ``` 
 
-## How to Use this Model
-
-The model is available for use in the NeMo toolkit [3], and can be used as a pre-trained checkpoint for inference or for fine-tuning on another dataset.
-
-### Automatically instantiate the model
+#### Automatically instantiate the model
 
 ```python
 import nemo.collections.asr as nemo_asr
 asr_model = nemo_asr.models.EncDecRNNTBPEModel.from_pretrained(model_name="nvidia/parakeet-rnnt-1.1b")
 ```
 
-### Transcribing using Python
+#### Transcribing using Python
 First, let's get a sample
 ```
 wget https://dldata-public.s3.us-east-2.amazonaws.com/2086-149220-0033.wav
@@ -213,7 +229,7 @@ output = asr_model.transcribe(['2086-149220-0033.wav'])
 print(output[0].text)
 ```
 
-### Transcribing many audio files
+#### Transcribing many audio files
 
 ```shell
 python [NEMO_GIT_FOLDER]/examples/asr/transcribe_speech.py 
@@ -221,6 +237,114 @@ python [NEMO_GIT_FOLDER]/examples/asr/transcribe_speech.py
  audio_dir="<DIRECTORY CONTAINING AUDIO FILES>"
 ```
 
+### 2) [Transformers](https://github.com/huggingface/transformers) 🤗 usage 
+
+
+Until Parakeet RNNT is part of an official Transformers release, you can use it by installing from source.
+
+```bash
+pip install git+https://github.com/huggingface/transformers
+```
+
+<details>
+  <summary>➡️ Pipeline usage</summary>
+
+```python
+from transformers import pipeline
+
+pipe = pipeline("automatic-speech-recognition", model="eustlb/parakeet-rnnt-1.1b")
+out = pipe("https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/bcn_weather.mp3")
+print(out)
+```
+</details>
+
+<details>
+  <summary>➡️ AutoModel</summary>
+
+```python
+from transformers import AutoModelForRNNT, AutoProcessor
+from datasets import load_dataset, Audio
+
+num_samples = 3
+
+model_id = "eustlb/parakeet-rnnt-1.1b"
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForRNNT.from_pretrained(model_id, dtype="auto", device_map="auto")
+
+ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))
+speech_samples = [el["array"] for el in ds["audio"][:num_samples]]
+
+inputs = processor(speech_samples, sampling_rate=processor.feature_extractor.sampling_rate)
+inputs.to(model.device, dtype=model.dtype)
+output = model.generate(**inputs, return_dict_in_generate=True)
+print(processor.decode(output.sequences, skip_special_tokens=True))
+```
+</details>
+
+
+<details>
+  <summary>➡️ Timestamping</summary>
+
+```python
+from datasets import Audio, load_dataset
+from transformers import AutoModelForRNNT, AutoProcessor
+
+num_samples = 3
+
+model_id = "eustlb/parakeet-rnnt-1.1b"
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForRNNT.from_pretrained(model_id, dtype="auto", device_map="auto")
+
+ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))
+speech_samples = [el["array"] for el in ds["audio"][:num_samples]]
+
+inputs = processor(speech_samples, sampling_rate=processor.feature_extractor.sampling_rate)
+inputs.to(model.device, dtype=model.dtype)
+output = model.generate(**inputs, return_dict_in_generate=True)
+decoded_output, decoded_timestamps = processor.decode(
+    output.sequences,
+    durations=output.durations,
+    skip_special_tokens=True,
+)
+print("Transcription:", decoded_output)
+print("Timestamped tokens:", decoded_timestamps)
+```
+</details>
+
+<details>
+  <summary>➡️ Training</summary>
+
+```python
+from transformers import AutoModelForRNNT, AutoProcessor
+from datasets import load_dataset, Audio
+import torch
+
+model_id = "eustlb/parakeet-rnnt-1.1b"
+NUM_SAMPLES = 4
+
+processor = AutoProcessor.from_pretrained(model_id)
+model = AutoModelForRNNT.from_pretrained(model_id, dtype=torch.bfloat16, device_map="auto")
+model.train()
+
+ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+ds = ds.cast_column("audio", Audio(sampling_rate=processor.feature_extractor.sampling_rate))
+speech_samples = [el["array"] for el in ds["audio"][:NUM_SAMPLES]]
+text_samples = ds["text"][:NUM_SAMPLES]
+
+# passing `text` to the processor will prepare inputs' `labels` key
+inputs = processor(audio=speech_samples, text=text_samples, sampling_rate=processor.feature_extractor.sampling_rate)
+inputs.to(device=model.device, dtype=model.dtype)
+
+outputs = model(**inputs)
+print("Loss:", outputs.loss.item())
+outputs.loss.backward()
+```
+</details>
+
+For more details about usage, please refer to the [Transformers' documentation](https://huggingface.co/docs/transformers/en/model_doc/parakeet).
+
 ### Input
 
 This model accepts 16000 Hz mono-channel audio (wav files) as input.