Fix audio placeholder length mismatch for merged audio inputs

processing_minicpmo.py

@@ -1039,16 +1039,16 @@ class MiniCPMOProcessor(ProcessorMixin):
         Returns:
             Audio placeholder string
         """
-        pool_step = self.pool_step
         feature_lens = math.ceil(audio_lens / self.audio_processor.hop_length)
-
         feature_lens = (feature_lens - 1) // 2 + 1
-        output_lens = (feature_lens - pool_step) // pool_step + 1
+        output_lens = (feature_lens - self.pool_step) // self.pool_step + 1
+        return self._get_audio_placeholder_by_output_lens(output_lens, chunk_input, chunk_length)
 
+    def _get_audio_placeholder_by_output_lens(self, output_lens: int, chunk_input: bool = True, chunk_length: int = 1):
         if chunk_input:
             fbank_feat_in_chunk = int(chunk_length * 100)
             cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
-            audio_embeds_in_chunk = (cnn_feat_in_chunk - pool_step) // pool_step + 1
+            audio_embeds_in_chunk = (cnn_feat_in_chunk - self.pool_step) // self.pool_step + 1
             num_audio_chunks = (output_lens + audio_embeds_in_chunk - 1) // audio_embeds_in_chunk
 
             place_holders = ""
@@ -1063,6 +1063,27 @@ class MiniCPMOProcessor(ProcessorMixin):
 
         return audio_placeholder
 
+    def _get_merged_audio_placeholders(self, audios, chunk_input: bool = True, chunk_length: int = 1):
+        audio_placeholders = []
+        prev_output_lens = 0
+        total_audio_lens = 0
+        for audio in audios:
+            total_audio_lens += len(audio)
+            output_lens = self.get_audio_placeholder(
+                total_audio_lens,
+                chunk_input=chunk_input,
+                chunk_length=chunk_length,
+            ).count("<unk>")
+            audio_placeholders.append(
+                self._get_audio_placeholder_by_output_lens(
+                    output_lens - prev_output_lens,
+                    chunk_input=chunk_input,
+                    chunk_length=chunk_length,
+                )
+            )
+            prev_output_lens = output_lens
+        return audio_placeholders
+
     def _init_streaming_processor(
         self,
         chunk_ms: int = 100,
@@ -1382,7 +1403,7 @@ class MiniCPMOProcessor(ProcessorMixin):
 
         if isinstance(audios, np.ndarray):
             audios_list = [[audios]]
-        elif isinstance(audios[0], np.ndarray):
+        elif len(audios) > 0 and isinstance(audios[0], np.ndarray):
             audios_list = [audios]
         else:
             audios_list = audios
@@ -1393,37 +1414,45 @@ class MiniCPMOProcessor(ProcessorMixin):
                 assert len(parts) == len(audios)
 
         audio_feature_lens_list = []
-        audio_ph_list = []
+        audio_ph_list = [[] for _ in audios_list]
         audio_features_all = []
 
-        # audio placeholder not dependent on audio_parts
-        for audios in audios_list:
-            if audios:
-                audio_ph_list.append(
-                    [
-                        self.get_audio_placeholder(len(a), chunk_input=stream_input, chunk_length=chunk_length)
-                        for a in audios
-                    ]
-                )
-            else:
-                audio_ph_list.append([])
-
         for idx, audios in enumerate(audios_list):
             if audio_parts is not None:
-                # same audio part merge
                 audio_part = audio_parts[idx]
                 merge_audio = []
                 cur_audio = []
+                audio_phs = []
                 for aid, (part, audio) in enumerate(zip(audio_part, audios)):
                     if aid == 0 or audio_part[aid] == audio_part[aid - 1]:
                         cur_audio.append(audio)
                     else:
                         merge_audio.append(np.hstack(cur_audio))
+                        audio_phs.extend(
+                            self._get_merged_audio_placeholders(
+                                cur_audio,
+                                chunk_input=stream_input,
+                                chunk_length=chunk_length,
+                            )
+                        )
                         cur_audio = [audio]
                 if cur_audio:
                     merge_audio.append(np.hstack(cur_audio))
+                    audio_phs.extend(
+                        self._get_merged_audio_placeholders(
+                            cur_audio,
+                            chunk_input=stream_input,
+                            chunk_length=chunk_length,
+                        )
+                    )
             else:
                 merge_audio = audios
+                audio_phs = [
+                    self.get_audio_placeholder(len(a), chunk_input=stream_input, chunk_length=chunk_length)
+                    for a in audios
+                ]
+
+            audio_ph_list[idx] = audio_phs
 
             # If the audio exceeds 30 seconds, split it into chunks every 30 seconds.
             final_merge_audio = []
@@ -1471,7 +1500,7 @@ class MiniCPMOProcessor(ProcessorMixin):
                     audio_feature_lens = torch.hstack(audio_feature_lens)
                     audio_feature_lens_list.append(audio_feature_lens)
             else:
-                audio_feature_lens_list.append([])
+                audio_feature_lens_list.append(torch.tensor([], dtype=torch.long))
 
         if audio_features_all:
             audio_features = [i.permute(1, 0) for i in audio_features_all]