Update models, simplify inputs

README.md

@@ -25,12 +25,39 @@ FunASR SenseVoice on Axera, official repo: https://github.com/FunAudioLLM/SenseV
 - [x] AX650N
 - [x] AX630C
 
+
 ## 环境安装
+
+推荐在板上安装Miniconda管理虚拟环境，安装方法如下:
 ```
-pip3 install -r requirements.txt
+mkdir -p ~/miniconda3
+wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh -O ~/miniconda3/miniconda.sh
+bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
+rm ~/miniconda3/miniconda.sh
+
+source ~/miniconda3/bin/activate
+
+conda init --all
+```
+
+```
+sudo apt-get install libsndfile-dev
+
+conda create -n sensevoice python=3.12
+conda activate sensevoice
+pip install -r requirements.txt
 ```
 如果空间不足可以使用 --prefix 指定别的安装路径
 
+####  安装pyaxenigne
+
+参考 https://github.com/AXERA-TECH/pyaxengine 安装 NPU Python API
+
+在0.1.3rc2上测试通过，可通过
+```
+pip install https://github.com/AXERA-TECH/pyaxengine/releases/download/0.1.3.rc2/axengine-0.1.3-py3-none-any.whl
+```
+安装，或把版本号更改为你想使用的版本
 
 ## 使用
 ```
@@ -48,20 +75,21 @@ python3 main.py -i 输入音频文件
 ### 示例:  
 example下有测试音频  
 
-如 粤语测试
+如 中文测试
 ```
-python3 main.py -i example/yue.mp3
+python main.py -i example/zh.mp3
 ```
 输出
 ```
-RTF: 0.03026517820946964    Latency: 0.15689468383789062s  Total length: 5.184s
-['呢几个字。', '都表达唔到，我想讲嘅意。', '思。']
+RTF: 0.04386647134764582    Latency: 0.2463541030883789s  Total length: 5.616s
+ASR result: 开饭时间早上九点至下午五点
+
 ```
 
 流式识别  
 
 ```
-python3 main.py -i example/zh.mp3 --streaming
+python main.py -i example/zh.mp3 --streaming
 ```
 输出
 ```
@@ -81,13 +109,13 @@ RTF: 0.03678379235444246
 
 使用WER(Word-Error-Rate)作为评价标准  
 
-**WER = 0.0389**  
+**WER = 2.0%**  
 
 ### 复现测试结果
 
 ```
 ./download_datasets.sh
-python test_wer.py -d datasets -l zh
+python test_wer.py -d aishell -g datasets/ground_truth.txt --language zh
 ```
 
 ## 技术讨论

SenseVoiceAx.py

@@ -2,44 +2,14 @@ import axengine as axe
 import numpy as np
 import librosa
 from frontend import WavFrontend
-import os
 import time
 from typing import List, Union, Optional
 from asr_decoder import CTCDecoder
-from tokenizer import SentencepiecesTokenizer
 from online_fbank import OnlineFbank
 import torch
 
 
-def sequence_mask(lengths, maxlen=None, dtype=np.float32):
-    # 如果 maxlen 未指定，则取 lengths 中的最大值
-    if maxlen is None:
-        maxlen = np.max(lengths)
-
-    # 创建一个从 0 到 maxlen-1 的行向量
-    row_vector = np.arange(0, maxlen, 1)
-
-    # 将 lengths 转换为列向量
-    matrix = np.expand_dims(lengths, axis=-1)
-
-    # 比较生成掩码
-    mask = row_vector < matrix
-    if mask.shape[-1] < lengths[0]:
-        mask = np.concatenate(
-            [
-                mask,
-                np.zeros(
-                    (mask.shape[0], lengths[0] - mask.shape[-1]), dtype=np.float32
-                ),
-            ],
-            axis=-1,
-        )
-
-    # 返回指定数据类型的掩码
-    return mask.astype(dtype)[None, ...]
-
-
-def unique_consecutive_np(arr):
+def unique_consecutive(arr):
     """
     找出数组中连续的唯一值，模拟 torch.unique_consecutive(yseq, dim=-1)
 
@@ -74,13 +44,14 @@ class SenseVoiceAx:
     def __init__(
         self,
         model_path: str,
-        max_len: int = 256,
+        cmvn_file: str,
+        token_file: str,
+        bpe_model: str = None,
+        max_seq_len: int = 256,
         beam_size: int = 3,
-        language: str = "auto",
         hot_words: Optional[List[str]] = None,
-        use_itn: bool = True,
         streaming: bool = False,
-        providers=['AxEngineExecutionProvider']
+        providers=["AxEngineExecutionProvider"],
     ):
         """
         Initialize SenseVoiceAx
@@ -99,23 +70,8 @@ class SenseVoiceAx:
                         Use stream_infer method if streaming is true otherwise infer.
 
         """
-        model_path_root = os.path.dirname(model_path)
-        emb_path = os.path.join(model_path_root, "../embeddings.npy")
-        cmvn_file = os.path.join(model_path_root, "../am.mvn")
-        bpe_model = os.path.join(
-            model_path_root, "../chn_jpn_yue_eng_ko_spectok.bpe.model"
-        )
-        if streaming:
-            self.position_encoding = np.load(
-                os.path.join(model_path_root, "../pe_streaming.npy")
-            )
-        else:
-            self.position_encoding = np.load(
-                os.path.join(model_path_root, "../pe_nonstream.npy")
-            )
 
         self.streaming = streaming
-        self.tokenizer = SentencepiecesTokenizer(bpemodel=bpe_model)
 
         self.frontend = WavFrontend(
             cmvn_file=cmvn_file,
@@ -127,12 +83,15 @@ class SenseVoiceAx:
             lfr_m=7,
             lfr_n=6,
         )
+
         self.model = axe.InferenceSession(model_path, providers=providers)
         self.sample_rate = 16000
         self.blank_id = 0
-        self.max_len = max_len
+        self.max_seq_len = max_seq_len
         self.padding = 16
         self.input_size = 560
+        self.query_num = 4
+        self.tokens = self.load_tokens(token_file)
 
         self.lid_dict = {
             "auto": 0,
@@ -143,33 +102,13 @@ class SenseVoiceAx:
             "ko": 12,
             "nospeech": 13,
         }
-        self.lid_int_dict = {
-            24884: 3,
-            24885: 4,
-            24888: 7,
-            24892: 11,
-            24896: 12,
-            24992: 13,
-        }
-        self.textnorm_dict = {"withitn": 14, "woitn": 15}
-        self.textnorm_int_dict = {25016: 14, 25017: 15}
-        self.emo_dict = {
-            "unk": 25009,
-            "happy": 25001,
-            "sad": 25002,
-            "angry": 25003,
-            "neutral": 25004,
-        }
-
-        self.load_embeddings(emb_path, language, use_itn)
-        self.language = language
 
         # decoder
         if beam_size > 1 and hot_words is not None:
             self.beam_size = beam_size
             symbol_table = {}
-            for i in range(self.tokenizer.get_vocab_size()):
-                symbol_table[self.tokenizer.decode(i)] = i
+            for i in range(len(self.tokens)):
+                symbol_table[self.tokens[i]] = i
             self.decoder = CTCDecoder(hot_words, symbol_table, bpe_model)
         else:
             self.beam_size = 1
@@ -177,8 +116,8 @@ class SenseVoiceAx:
 
         if streaming:
             self.cur_idx = -1
-            self.chunk_size = max_len - self.padding
-            self.caches_shape = (max_len, self.input_size)
+            self.chunk_size = max_seq_len - self.padding
+            self.caches_shape = (max_seq_len, self.input_size)
             self.caches = np.zeros(self.caches_shape, dtype=np.float32)
             self.zeros = np.zeros((1, self.input_size), dtype=np.float32)
             self.neg_mean, self.inv_stddev = (
@@ -187,38 +126,25 @@ class SenseVoiceAx:
             )
 
             self.fbank = OnlineFbank(window_type="hamming")
-            self.masks = sequence_mask(
-                np.array([self.max_len], dtype=np.int32),
-                maxlen=self.max_len,
-                dtype=np.float32,
+            self.stream_mask = self.sequence_mask(
+                max_seq_len + self.query_num, max_seq_len + self.query_num
             )
 
+    def load_tokens(self, token_file):
+        tokens = []
+        with open(token_file, "r") as f:
+            for line in f:
+                tokens.append(line[:-1])
+        return tokens
+
     @property
     def language_options(self):
         return list(self.lid_dict.keys())
 
-    @property
-    def textnorm_options(self):
-        return list(self.textnorm_dict.keys())
-
-    def load_embeddings(self, emb_path, language, use_itn):
-        self.embeddings = np.load(emb_path, allow_pickle=True).item()
-        self.language_query = self.embeddings[language]
-        self.textnorm_query = (
-            self.embeddings["withitn"] if use_itn else self.embeddings["woitn"]
-        )
-        self.event_emo_query = self.embeddings["event_emo"]
-        self.input_query = np.concatenate(
-            (self.textnorm_query, self.language_query, self.event_emo_query), axis=1
-        )
-        self.query_num = self.input_query.shape[1]
-
-    def choose_language(self, language):
-        self.language_query = self.embeddings[language]
-        self.input_query = np.concatenate(
-            (self.textnorm_query, self.language_query, self.event_emo_query), axis=1
-        )
-        self.language = language
+    def sequence_mask(self, max_seq_len, actual_seq_len):
+        mask = np.zeros((1, 1, max_seq_len), dtype=np.int32)
+        mask[:, :, :actual_seq_len] = 1
+        return mask
 
     def load_data(self, filepath: str) -> np.ndarray:
         waveform, _ = librosa.load(filepath, sr=self.sample_rate)
@@ -254,7 +180,7 @@ class SenseVoiceAx:
         yseq = np.argmax(x, axis=-1)
 
         # 去除连续重复元素
-        yseq = unique_consecutive_np(yseq)
+        yseq = unique_consecutive(yseq)
 
         # 创建掩码并过滤 blank_id
         mask = yseq != self.blank_id
@@ -263,16 +189,16 @@ class SenseVoiceAx:
         return token_int
 
     def infer_waveform(self, waveform: np.ndarray, language="auto"):
-        if language != self.language:
-            self.choose_language(language)
-
         # start = time.time()
         feat, feat_len = self.preprocess(waveform)
         # print(f"Preprocess take {time.time() - start}s")
 
-        slice_len = self.max_len - self.query_num
+        slice_len = self.max_seq_len - self.query_num
         slice_num = int(np.ceil(feat.shape[1] / slice_len))
 
+        language_token = self.lid_dict[language]
+        language_token = np.array([language_token], dtype=np.int32)
+
         asr_res = []
         for i in range(slice_num):
             if i == 0:
@@ -283,46 +209,39 @@ class SenseVoiceAx:
                     i * slice_len - self.padding : (i + 1) * slice_len - self.padding,
                     :,
                 ]
-            # concat query
-            sub_feat = np.concatenate([self.input_query, sub_feat], axis=1)
+
             real_len = sub_feat.shape[1]
-            if real_len < self.max_len:
+            if real_len < self.max_seq_len:
                 sub_feat = np.concatenate(
                     [
                         sub_feat,
                         np.zeros(
-                            (1, self.max_len - real_len, sub_feat.shape[-1]),
+                            (1, self.max_seq_len - real_len, sub_feat.shape[-1]),
                             dtype=np.float32,
                         ),
                     ],
                     axis=1,
                 )
 
-            masks = sequence_mask(
-                np.array([self.max_len], dtype=np.int32),
-                maxlen=real_len,
-                dtype=np.float32,
-            )
+            mask = self.sequence_mask(self.max_seq_len + self.query_num, real_len)
 
             # start = time.time()
             outputs = self.model.run(
                 None,
                 {
                     "speech": sub_feat,
-                    "masks": masks,
-                    "position_encoding": self.position_encoding,
+                    "mask": mask,
+                    "language": language_token,
                 },
             )
             ctc_logits, encoder_out_lens = outputs
 
             token_int = self.postprocess(ctc_logits, encoder_out_lens)
 
-            if self.tokenizer is not None:
-                asr_res.append(self.tokenizer.tokens2text(token_int))
-            else:
-                asr_res.append(token_int)
+            asr_res.extend(token_int)
 
-        return asr_res
+        text = "".join([self.tokens[i] for i in asr_res])
+        return text
 
     def infer(
         self, filepath_or_data: Union[np.ndarray, str], language="auto", print_rtf=False
@@ -343,15 +262,15 @@ class SenseVoiceAx:
         if print_rtf:
             rtf = latency / total_time
             print(f"RTF: {rtf}    Latency: {latency}s  Total length: {total_time}s")
-        return "".join(asr_res)
+        return asr_res
 
     def decode(self, times, tokens):
         times_ms = []
         for step, token in zip(times, tokens):
-            if len(self.tokenizer.decode(token).strip()) == 0:
+            if len(self.tokens[token].strip()) == 0:
                 continue
             times_ms.append(step * 60)
-        return times_ms, self.tokenizer.decode(tokens)
+        return times_ms, "".join([self.tokens[i] for i in tokens])
 
     def reset(self):
         self.cur_idx = -1
@@ -368,8 +287,8 @@ class SenseVoiceAx:
     def stream_infer(self, audio, is_last, language="auto"):
         assert self.streaming, "This method is for streaming model"
 
-        if language != self.language:
-            self.choose_language(language)
+        language_token = self.lid_dict[language]
+        language_token = np.array([language_token], dtype=np.int32)
 
         self.fbank.accept_waveform(audio, is_last)
         features = self.fbank.get_lfr_frames(
@@ -393,8 +312,8 @@ class SenseVoiceAx:
                 None,
                 {
                     "speech": speech,
-                    "masks": self.masks,
-                    "position_encoding": self.position_encoding,
+                    "mask": self.stream_mask,
+                    "language": language_token,
                 },
             )
             ctc_logits, encoder_out_lens = outputs

gradio_demo.py

@@ -1,25 +1,31 @@
 import gradio as gr
 import os
 from SenseVoiceAx import SenseVoiceAx
-from print_utils import rich_transcription_postprocess
+from download_utils import download_model
 
-max_len = 256
-
-model_path = os.path.join("sensevoice_ax650", "sensevoice.axmodel")
+model_root = download_model("SenseVoice")
+model_root = os.path.join(model_root, "sensevoice_ax650")
+max_seq_len = 256
+model_path = os.path.join(model_root, "sensevoice.axmodel")
 
 assert os.path.exists(model_path), f"model {model_path} not exist"
 
-pipeline = SenseVoiceAx(
+cmvn_file = os.path.join(model_root, "am.mvn")
+bpe_model = os.path.join(model_root, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+token_file = os.path.join(model_root, "tokens.txt")
+
+model = SenseVoiceAx(
     model_path,
-    max_len=max_len,
+    cmvn_file,
+    token_file,
+    bpe_model,
+    max_seq_len=max_seq_len,
     beam_size=3,
-    language="auto",
     hot_words=None,
-    use_itn=True,
     streaming=False,
 )
 
-
+# 你实现的语言转文本函数
 def speech_to_text(audio_path, lang):
     """
     audio_path: 音频文件路径
@@ -28,9 +34,7 @@ def speech_to_text(audio_path, lang):
     if not audio_path:
         return "无音频"
 
-    pipeline.choose_language(language=lang)
-    asr_res = pipeline.infer(audio_path, print_rtf=False)
-
+    asr_res = model.infer(audio_path, lang, print_rtf=False)
     return asr_res
 
 
@@ -41,7 +45,7 @@ def main():
 
         with gr.Row():
             audio_input = gr.Audio(
-                sources=["upload"], type="filepath", label="录制或上传音频", format="mp3"
+                sources=["microphone"], type="filepath", label="录制或上传音频", format="mp3"
             )
             lang_dropdown = gr.Dropdown(
                 choices=["auto", "zh", "en", "yue", "ja", "ko"],
@@ -55,6 +59,10 @@ def main():
 
     demo.launch(
         server_name="0.0.0.0",
+        server_port=7860,
+        ssl_certfile="./cert.pem",
+        ssl_keyfile="./key.pem",
+        ssl_verify=False,
     )
 
 

main.py

@@ -2,7 +2,7 @@ import os
 import argparse
 from SenseVoiceAx import SenseVoiceAx
 import librosa
-import numpy as np
+from download_utils import download_model
 import time
 
 
@@ -25,37 +25,38 @@ def get_args():
 
 def main():
     args = get_args()
+    print(vars(args))
 
     input_audio = args.input
     language = args.language
-    use_itn = True  # 标点符号预测
+    model_root = download_model("SenseVoice")
+    model_root = os.path.join(model_root, "sensevoice_ax650")
     if not args.streaming:
-        max_len = 256
-        model_path = os.path.join("sensevoice_ax650", "sensevoice.axmodel")
+        max_seq_len = 256
+        model_path = os.path.join(model_root, "sensevoice.axmodel")
     else:
-        max_len = 26
-        model_path = os.path.join("sensevoice_ax650", "streaming_sensevoice.axmodel")
+        max_seq_len = 26
+        model_path = os.path.join(model_root, "streaming_sensevoice.axmodel")
 
     assert os.path.exists(model_path), f"model {model_path} not exist"
 
-    print(f"input_audio: {input_audio}")
-    print(f"language: {language}")
-    print(f"use_itn: {use_itn}")
-    print(f"model_path: {model_path}")
-    print(f"streaming: {args.streaming}")
+    cmvn_file = os.path.join(model_root, "am.mvn")
+    bpe_model = os.path.join(model_root, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+    token_file = os.path.join(model_root, "tokens.txt")
 
-    pipeline = SenseVoiceAx(
+    model = SenseVoiceAx(
         model_path,
-        max_len=max_len,
+        cmvn_file,
+        token_file,
+        bpe_model,
+        max_seq_len=max_seq_len,
         beam_size=3,
-        language="auto",
         hot_words=None,
-        use_itn=True,
         streaming=args.streaming,
     )
 
     if not args.streaming:
-        asr_res = pipeline.infer(input_audio, print_rtf=True)
+        asr_res = model.infer(input_audio, language, print_rtf=True)
         print("ASR result: " + asr_res)
     else:
         samples, sr = librosa.load(input_audio, sr=16000)
@@ -66,7 +67,7 @@ def main():
         step = int(0.1 * sr)
         for i in range(0, len(samples), step):
             is_last = i + step >= len(samples)
-            for res in pipeline.stream_infer(samples[i : i + step], is_last):
+            for res in model.stream_infer(samples[i : i + step], is_last, language):
                 print(res)
 
         end = time.time()

pe_nonstream.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0f1c9c550bd62fa164a959517f52d46a28591812fafdf002df0df2bd998f44b5
-size 573568

pe_streaming.npy

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:54fec2fe2670168d36678c5857e65c459c634e6b6d6df928b7d415399ce2c291
-size 58368

print_utils.py

@@ -1,131 +0,0 @@
-emo_dict = {
-    "<|HAPPY|>": "😊",
-    "<|SAD|>": "😔",
-    "<|ANGRY|>": "😡",
-    "<|NEUTRAL|>": "",
-    "<|FEARFUL|>": "😰",
-    "<|DISGUSTED|>": "🤢",
-    "<|SURPRISED|>": "😮",
-}
-
-event_dict = {
-    "<|BGM|>": "🎼",
-    "<|Speech|>": "",
-    "<|Applause|>": "👏",
-    "<|Laughter|>": "😀",
-    "<|Cry|>": "😭",
-    "<|Sneeze|>": "🤧",
-    "<|Breath|>": "",
-    "<|Cough|>": "🤧",
-}
-
-lang_dict = {
-    "<|zh|>": "<|lang|>",
-    "<|en|>": "<|lang|>",
-    "<|yue|>": "<|lang|>",
-    "<|ja|>": "<|lang|>",
-    "<|ko|>": "<|lang|>",
-    "<|nospeech|>": "<|lang|>",
-}
-
-emoji_dict = {
-    "<|nospeech|><|Event_UNK|>": "❓",
-    "<|zh|>": "",
-    "<|en|>": "",
-    "<|yue|>": "",
-    "<|ja|>": "",
-    "<|ko|>": "",
-    "<|nospeech|>": "",
-    "<|HAPPY|>": "😊",
-    "<|SAD|>": "😔",
-    "<|ANGRY|>": "😡",
-    "<|NEUTRAL|>": "",
-    "<|BGM|>": "🎼",
-    "<|Speech|>": "",
-    "<|Applause|>": "👏",
-    "<|Laughter|>": "😀",
-    "<|FEARFUL|>": "😰",
-    "<|DISGUSTED|>": "🤢",
-    "<|SURPRISED|>": "😮",
-    "<|Cry|>": "😭",
-    "<|EMO_UNKNOWN|>": "",
-    "<|Sneeze|>": "🤧",
-    "<|Breath|>": "",
-    "<|Cough|>": "😷",
-    "<|Sing|>": "",
-    "<|Speech_Noise|>": "",
-    "<|withitn|>": "",
-    "<|woitn|>": "",
-    "<|GBG|>": "",
-    "<|Event_UNK|>": "",
-}
-
-emo_set = {"😊", "😔", "😡", "😰", "🤢", "😮"}
-event_set = {
-    "🎼",
-    "👏",
-    "😀",
-    "😭",
-    "🤧",
-    "😷",
-}
-
-
-def format_str_v2(s):
-    sptk_dict = {}
-    for sptk in emoji_dict:
-        sptk_dict[sptk] = s.count(sptk)
-        s = s.replace(sptk, "")
-    emo = "<|NEUTRAL|>"
-    for e in emo_dict:
-        if sptk_dict[e] > sptk_dict[emo]:
-            emo = e
-    for e in event_dict:
-        if sptk_dict[e] > 0:
-            s = event_dict[e] + s
-    s = s + emo_dict[emo]
-
-    for emoji in emo_set.union(event_set):
-        s = s.replace(" " + emoji, emoji)
-        s = s.replace(emoji + " ", emoji)
-    return s.strip()
-
-
-def rich_transcription_postprocess(s):
-    def get_emo(s):
-        return s[-1] if s[-1] in emo_set else None
-
-    def get_event(s):
-        return s[0] if s[0] in event_set else None
-
-    s = s.replace("<|nospeech|><|Event_UNK|>", "❓")
-    for lang in lang_dict:
-        s = s.replace(lang, "<|lang|>")
-    s_list = [format_str_v2(s_i).strip(" ") for s_i in s.split("<|lang|>")]
-    new_s = " " + s_list[0]
-    cur_ent_event = get_event(new_s)
-    for i in range(1, len(s_list)):
-        if len(s_list[i]) == 0:
-            continue
-        if get_event(s_list[i]) == cur_ent_event and get_event(s_list[i]) != None:
-            s_list[i] = s_list[i][1:]
-        # else:
-        cur_ent_event = get_event(s_list[i])
-        if get_emo(s_list[i]) != None and get_emo(s_list[i]) == get_emo(new_s):
-            new_s = new_s[:-1]
-        new_s += s_list[i].strip().lstrip()
-    new_s = new_s.replace("The.", " ")
-    return new_s.strip()
-
-
-def rich_print_asr_res(asr_res, will_print=True, remove_punc=False):
-    res = "".join([rich_transcription_postprocess(i) for i in asr_res])
-
-    if remove_punc:
-        res = res.replace("，", "")
-        res = res.replace("。", "")
-
-    if will_print:
-        print(res)
-
-    return res

requirements.txt

@@ -2,10 +2,7 @@ huggingface_hub
 numpy<2
 kaldi-native-fbank
 librosa==0.9.1
-sentencepiece
 fastapi
 gradio
-emoji
-asr-decoder
 online-fbank
-torch
+asr_decoder

sensevoice_ax630c/sensevoice.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67d290cf7cebf45db5f37b2e93b8bdfff44dc35110bb29d84204a5f9eae9fd4d
-size 256550253
+oid sha256:bdcac5038b7062719a19bed49f39e448e9d741ec389fb1c9b0c62d9efb5a1a8e
+size 259948631

sensevoice_ax630c/streaming_sensevoice.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba1ddd60841297903bfdae059ad88092d0fd1c543e1d80d7f64199d4e27b8263
-size 249023211
+oid sha256:607af1407270dfdff95421286d29286aaab4d93885332d5a6f84810b1042fb2b
+size 249359616

sensevoice_ax650/am.mvn

@@ -0,0 +1,8 @@
+<Nnet> 
+<Splice> 560 560
+[ 0 ]
+<AddShift> 560 560 
+<LearnRateCoef> 0 [ -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 -8.311879 -8.600912 -9.615928 -10.43595 -11.21292 -11.88333 -12.36243 -12.63706 -12.8818 -12.83066 -12.89103 -12.95666 -13.19763 -13.40598 -13.49113 -13.5546 -13.55639 -13.51915 -13.68284 -13.53289 -13.42107 -13.65519 -13.50713 -13.75251 -13.76715 -13.87408 -13.73109 -13.70412 -13.56073 -13.53488 -13.54895 -13.56228 -13.59408 -13.62047 -13.64198 -13.66109 -13.62669 -13.58297 -13.57387 -13.4739 -13.53063 -13.48348 -13.61047 -13.64716 -13.71546 -13.79184 -13.90614 -14.03098 -14.18205 -14.35881 -14.48419 -14.60172 -14.70591 -14.83362 -14.92122 -15.00622 -15.05122 -15.03119 -14.99028 -14.92302 -14.86927 -14.82691 -14.7972 -14.76909 -14.71356 -14.61277 -14.51696 -14.42252 -14.36405 -14.30451 -14.23161 -14.19851 -14.16633 -14.15649 -14.10504 -13.99518 -13.79562 -13.3996 -12.7767 -11.71208 ]
+<Rescale> 560 560
+<LearnRateCoef> 0 [ 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 0.155775 0.154484 0.1527379 0.1518718 0.1506028 0.1489256 0.147067 0.1447061 0.1436307 0.1443568 0.1451849 0.1455157 0.1452821 0.1445717 0.1439195 0.1435867 0.1436018 0.1438781 0.1442086 0.1448844 0.1454756 0.145663 0.146268 0.1467386 0.1472724 0.147664 0.1480913 0.1483739 0.1488841 0.1493636 0.1497088 0.1500379 0.1502916 0.1505389 0.1506787 0.1507102 0.1505992 0.1505445 0.1505938 0.1508133 0.1509569 0.1512396 0.1514625 0.1516195 0.1516156 0.1515561 0.1514966 0.1513976 0.1512612 0.151076 0.1510596 0.1510431 0.151077 0.1511168 0.1511917 0.151023 0.1508045 0.1505885 0.1503493 0.1502373 0.1501726 0.1500762 0.1500065 0.1499782 0.150057 0.1502658 0.150469 0.1505335 0.1505505 0.1505328 0.1504275 0.1502438 0.1499674 0.1497118 0.1494661 0.1493102 0.1493681 0.1495501 0.1499738 0.1509654 ]
+</Nnet> 

embeddings.npy → sensevoice_ax650/chn_jpn_yue_eng_ko_spectok.bpe.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a453244ab037744531b97bcb8574c8442301dac11f6406fdab208dddb83b93e
-size 25523
+oid sha256:aa87f86064c3730d799ddf7af3c04659151102cba548bce325cf06ba4da4e6a8
+size 377341

sensevoice_ax650/sensevoice.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fad2f710930c23c91ea62d6951c0c6161194e3cf356fc31611798419c6638dd9
-size 262381979
+oid sha256:91b0dcf88b85af852c4ca16e3879b317272bc748c9815cd91007cf71a0c59714
+size 263172727

sensevoice_ax650/streaming_sensevoice.axmodel

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9a6902048563d4f3b0442e380187f1b9d840bdcec476fceb158fe45d0cc12067
-size 261450261
+oid sha256:c883c2988e9bc7a103ac332611523f20bd474c3b10cc35e3cc2e621d35097756
+size 261538678

server.py

@@ -3,7 +3,9 @@ from fastapi import FastAPI, HTTPException, Body
 from fastapi.responses import JSONResponse
 from typing import List, Optional
 import logging
+import json
 from SenseVoiceAx import SenseVoiceAx
+from download_utils import download_model
 import os
 import librosa
 
@@ -28,27 +30,35 @@ async def load_model():
     try:
         # 模型加载
         language = "auto"
-        use_itn = True  # 逆文本规范
-        max_len = 256
+        use_itn = True  # 标点符号预测
+        max_len = 68
 
-        model_path = os.path.join("sensevoice_ax650", "sensevoice.axmodel")
+        model_root = download_model("SenseVoice")
+        model_root = os.path.join(model_root, "sensevoice_ax650")
+        max_seq_len = 256
+        model_path = os.path.join(model_root, "sensevoice.axmodel")
 
         assert os.path.exists(model_path), f"model {model_path} not exist"
 
-        print(f"language: {language}")
-        print(f"use_itn: {use_itn}")
-        print(f"model_path: {model_path}")
+        cmvn_file = os.path.join(model_root, "am.mvn")
+        bpe_model = os.path.join(model_root, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+        token_file = os.path.join(model_root, "tokens.txt")
 
         asr_model = SenseVoiceAx(
             model_path,
-            max_len=max_len,
+            cmvn_file,
+            token_file,
+            bpe_model,
+            max_seq_len=max_seq_len,
             beam_size=3,
-            language="auto",
             hot_words=None,
-            use_itn=use_itn,
             streaming=False,
         )
 
+        print(f"language: {language}")
+        print(f"use_itn: {use_itn}")
+        print(f"model_path: {model_path}")
+
         logger.info("ASR model loaded successfully")
     except Exception as e:
         logger.error(f"Failed to load ASR model: {str(e)}")

test_wer.py

@@ -1,12 +1,9 @@
-import os, sys
+import os
 import argparse
 from SenseVoiceAx import SenseVoiceAx
-from tokenizer import SentencepiecesTokenizer
-from print_utils import rich_transcription_postprocess, rich_print_asr_res
 from download_utils import download_model
 import logging
 import re
-import emoji
 
 
 def setup_logging():
@@ -229,7 +226,6 @@ def main():
     args = get_args()
 
     language = args.language
-    use_itn = False  # 标点符号预测
     max_num = args.max_num
 
     dataset_type = args.dataset.lower()
@@ -240,21 +236,32 @@ def main():
     else:
         raise ValueError(f"Unknown dataset type {dataset_type}")
 
-    # model_path_root = download_model("SenseVoice")
-    model_path = os.path.join("sensevoice_ax650", "sensevoice.axmodel")
-    bpemodel = "chn_jpn_yue_eng_ko_spectok.bpe.model"
+    model_root = download_model("SenseVoice")
+    model_root = os.path.join(model_root, "sensevoice_ax650")
+    max_seq_len = 256
+    model_path = os.path.join(model_root, "sensevoice.axmodel")
 
     assert os.path.exists(model_path), f"model {model_path} not exist"
 
+    cmvn_file = os.path.join(model_root, "am.mvn")
+    bpe_model = os.path.join(model_root, "chn_jpn_yue_eng_ko_spectok.bpe.model")
+    token_file = os.path.join(model_root, "tokens.txt")
+
+    model = SenseVoiceAx(
+        model_path,
+        cmvn_file,
+        token_file,
+        bpe_model,
+        max_seq_len=max_seq_len,
+        beam_size=3,
+        hot_words=None,
+        streaming=False,
+    )
+
     logger.info(f"dataset: {args.dataset}")
     logger.info(f"language: {language}")
-    logger.info(f"use_itn: {use_itn}")
     logger.info(f"model_path: {model_path}")
 
-    pipeline = SenseVoiceAx(
-        model_path, language=language
-    )
-
     # Iterate over dataset
     hyp = []
     references = []
@@ -264,11 +271,8 @@ def main():
     for n, (audio_path, reference) in enumerate(dataset):
         reference = remove_punctuation(reference).lower()
 
-        asr_res = pipeline.infer(audio_path, print_rtf=False)
-        hypothesis = rich_print_asr_res(
-            asr_res, will_print=False, remove_punc=True
-        ).lower()
-        hypothesis = emoji.replace_emoji(hypothesis, replace="")
+        asr_res = model.infer(audio_path, language, print_rtf=False)
+        hypothesis = remove_punctuation(asr_res).lower()
 
         character_error_num = min_distance(reference, hypothesis)
         character_num = len(reference)

tokenizer.py

@@ -1,135 +0,0 @@
-import sentencepiece as spm
-
-from pathlib import Path
-from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union
-
-import json
-from abc import abstractmethod
-from abc import ABC
-import numpy as np
-
-
-class BaseTokenizer(ABC):
-    def __init__(
-        self,
-        token_list: Union[Path, str, Iterable[str]] = None,
-        unk_symbol: str = "<unk>",
-        **kwargs,
-    ):
-
-        if token_list is not None:
-            if isinstance(token_list, (Path, str)) and token_list.endswith(".txt"):
-                token_list = Path(token_list)
-                self.token_list_repr = str(token_list)
-                self.token_list: List[str] = []
-
-                with token_list.open("r", encoding="utf-8") as f:
-                    for idx, line in enumerate(f):
-                        line = line.rstrip()
-                        self.token_list.append(line)
-            elif isinstance(token_list, (Path, str)) and token_list.endswith(".json"):
-                token_list = Path(token_list)
-                self.token_list_repr = str(token_list)
-                self.token_list: List[str] = []
-
-                with open(token_list, "r", encoding="utf-8") as f:
-                    self.token_list = json.load(f)
-
-            else:
-                self.token_list: List[str] = list(token_list)
-                self.token_list_repr = ""
-                for i, t in enumerate(self.token_list):
-                    if i == 3:
-                        break
-                    self.token_list_repr += f"{t}, "
-                self.token_list_repr += f"... (NVocab={(len(self.token_list))})"
-
-            self.token2id: Dict[str, int] = {}
-            for i, t in enumerate(self.token_list):
-                if t in self.token2id:
-                    raise RuntimeError(f'Symbol "{t}" is duplicated')
-                self.token2id[t] = i
-
-            self.unk_symbol = unk_symbol
-            if self.unk_symbol not in self.token2id:
-                raise RuntimeError(
-                    f"Unknown symbol '{unk_symbol}' doesn't exist in the token_list"
-                )
-            self.unk_id = self.token2id[self.unk_symbol]
-
-    def encode(self, text, **kwargs):
-        tokens = self.text2tokens(text)
-        text_ints = self.tokens2ids(tokens)
-
-        return text_ints
-
-    def decode(self, text_ints):
-        token = self.ids2tokens(text_ints)
-        text = self.tokens2text(token)
-        return text
-
-    def get_num_vocabulary_size(self) -> int:
-        return len(self.token_list)
-
-    def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]:
-        if isinstance(integers, np.ndarray) and integers.ndim != 1:
-            raise ValueError(f"Must be 1 dim ndarray, but got {integers.ndim}")
-        return [self.token_list[i] for i in integers]
-
-    def tokens2ids(self, tokens: Iterable[str]) -> List[int]:
-        return [self.token2id.get(i, self.unk_id) for i in tokens]
-
-    @abstractmethod
-    def text2tokens(self, line: str) -> List[str]:
-        raise NotImplementedError
-
-    @abstractmethod
-    def tokens2text(self, tokens: Iterable[str]) -> str:
-        raise NotImplementedError
-
-
-class SentencepiecesTokenizer(BaseTokenizer):
-    def __init__(self, bpemodel: Union[Path, str], **kwargs):
-        super().__init__(**kwargs)
-        self.bpemodel = str(bpemodel)
-        # NOTE(kamo):
-        # Don't build SentencePieceProcessor in __init__()
-        # because it's not picklable and it may cause following error,
-        # "TypeError: can't pickle SwigPyObject objects",
-        # when giving it as argument of "multiprocessing.Process()".
-        self.sp = None
-        self._build_sentence_piece_processor()
-
-    def __repr__(self):
-        return f'{self.__class__.__name__}(model="{self.bpemodel}")'
-
-    def _build_sentence_piece_processor(self):
-        # Build SentencePieceProcessor lazily.
-        if self.sp is None:
-            self.sp = spm.SentencePieceProcessor()
-            self.sp.load(self.bpemodel)
-
-    def text2tokens(self, line: str) -> List[str]:
-        self._build_sentence_piece_processor()
-        return self.sp.EncodeAsPieces(line)
-
-    def tokens2text(self, tokens: Iterable[str]) -> str:
-        self._build_sentence_piece_processor()
-        return self.sp.DecodePieces(list(tokens))
-
-    def encode(self, line: str, **kwargs) -> List[int]:
-        self._build_sentence_piece_processor()
-        return self.sp.EncodeAsIds(line)
-
-    def decode(self, line: List[int], **kwargs):
-        self._build_sentence_piece_processor()
-        return self.sp.DecodeIds(line)
-
-    def get_vocab_size(self):
-        return self.sp.GetPieceSize()
-
-    def ids2tokens(self, *args, **kwargs):
-        return self.decode(*args, **kwargs)
-
-    def tokens2ids(self, *args, **kwargs):
-        return self.encode(*args, **kwargs)