add some code

managed_components/espressif__esp-sr/tool/README.md

@@ -0,0 +1,67 @@
+## MultiNet6
+
+#### Step 1. Data preparation
+
+For English, words are used as units. Please prepare a list of commands written in a text file `commands_en.txt` of the following format:
+
+```
+# command_id command_sentence
+1 TELL ME A JOKE
+2 MAKE A COFFEE
+```
+
+For Chinese, pinyin are used as units. [multinet_pinyin.py](./multinet_pinyin.py) help to get Pinyin of Chinese. Please prepare a list of commands written in a text file `commands_cn.txt` of the following format:
+```
+# command_id command_sentence
+1 da kai kong tiao
+2 guan bi kong tiao
+```
+
+#### Step 2. Move created files
+
+1. Move your `commands_en.txt` or `commands_cn.txt` to `/model/multinet_model/fst/`
+
+## MultiNet5
+####  1. Install g2p_en, please refer to https://pypi.org/project/g2p-en/ 
+
+```
+pip install g2p_en
+```
+
+#### 2. Run multinet_g2p.py
+
+```
+python multinet_g2p.py -t "hello world,hi ESP;turn on the light;turn off the light"
+
+------
+in: hello world,hi ESP;turn on the light;turn off the light
+out: hcLb WkLD,hi fST;TkN nN jc LiT;TkN eF jc LiT;
+```
+
+#### 3. Add speech commands 
+
+##### 3.1 add speech commands by menuconfig
+
+```
+idf.py menuconfig
+ESP Speech Recognition -> Add speech commands
+```
+
+
+
+##### 3.2 add speech commands by reset function
+
+```
+// Function definition
+// typedef void (*esp_mn_iface_op_reset_t)(model_iface_data_t *model_data, char *command_str, char *err_phrase_id);
+
+// "," is used to split different phrase with same command id
+// ";" is used to split different command id
+char *new_commands_str="hcLb WkLD,hi fST;TkN nN jc LiT;TkN eF jc LiT;"  //
+char err_id[256];
+multinet->reset(model_data, new_commands_str, err_id);
+// hello world,hi ESP -> commond id=0
+// turn on the light -> commond id=1
+// turn off the light -> commond id=2
+```
+

managed_components/espressif__esp-sr/tool/fst/commands_list.txt

@@ -0,0 +1,49 @@
+1 TELL ME A JOKE
+2 SING A SONG
+3 PLAY NEWS CHANNEL
+4 TURN ON MY SOUNDBOX
+5 TURN OFF MY SOUNDBOX
+5 TURN OF MY SOUNDBOX
+6 HIGHEST VOLUME
+7 LOWEST VOLUME
+8 INCREASE THE VOLUME
+9 DECREASE THE VOLUME
+10 TURN ON THE TV
+11 TURN OFF THE TV
+11 TURN OF THE TV
+12 MAKE ME A TEA
+13 MAKE ME A COFFEE
+14 TURN ON THE LIGHT
+15 TURN OFF THE LIGHT
+15 TURN OF THE LIGHT
+16 CHANGE THE COLOR TO RED
+17 CHANGE THE COLOR TO GREEN
+18 TURN ON ALL THE LIGHTS
+19 TURN OFF ALL THE LIGHTS
+19 TURN OF ALL THE LIGHTS
+20 TURN ON THE AIR CONDITIONER
+21 TURN OFF THE AIR CONDITIONER
+21 TURN OF THE AIR CONDITIONER
+22 SET THE TEMPERATURE TO SIXTEEN DEGREES
+23 SET THE TEMPERATURE TO SEVENTEEN DEGREES
+24 SET THE TEMPERATURE TO EIGHTEEN DEGREES
+25 SET THE TEMPERATURE TO NINETEEN DEGREES
+26 SET THE TEMPERATURE TO TWENTY DEGREES
+27 SET THE TEMPERATURE TO TWENTY ONE DEGREES
+28 SET THE TEMPERATURE TO TWENTY TWO DEGREES
+29 SET THE TEMPERATURE TO TWENTY THREE DEGREES
+30 SET THE TEMPERATURE TO TWENTY FOUR DEGREES
+31 SET THE TEMPERATURE TO TWENTY FIVE DEGREES
+32 SET THE TEMPERATURE TO TWENTY SIX DEGREES
+33 LOWEST FAN SPEED
+34 MEDIUM FAN SPEED
+35 HIGHEST FAN SPEED
+36 AUTO ADJUST THE FAN SPEED
+37 DECREASE THE FAN SPEED
+38 INCREASE THE FAN SPEED
+39 INCREASE THE TEMPERATURE
+40 DECREASE THE TEMPERATURE
+41 COOLING MODE
+42 HEATING MODE
+43 VENTILATION MODE
+44 DEHUMIDIFY MODE

managed_components/espressif__esp-sr/tool/fst/prepare_for_fst.py

@@ -0,0 +1,64 @@
+import argparse
+from pathlib import Path
+from typing import List, Set, Tuple
+
+import sentencepiece as spm
+
+
+def process_commands(infile: List[str], sp: spm.SentencePieceProcessor
+                     ) -> Tuple[List[str], Set[int]]:
+    out_commands = []
+    tokens = set()
+
+    for line in infile:
+        command_id = line.split()[0]
+        command = ' '.join(line.split()[1:])
+        command_tokens = sp.encode(command, out_type=str)
+        for token in command_tokens:
+            tokens.add(token)
+        command_tokens = [command_id] + command_tokens
+        out_commands.append('\t'.join(command_tokens))
+    return out_commands, tokens
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--infile', type=str, required=True,
+                        help='the text file of commands id and commands.')
+    parser.add_argument('--bpe-model', type=str, default='bpe.model',
+                        help='subword bpe model file.')
+    parser.add_argument('--out-command-list', type=str,
+                        default='commands_tokens.txt',
+                        help='the output subword commands text filename.')
+    parser.add_argument('--out-token-symbols', type=str,
+                        default='tokens.txt',
+                        help='the output token to subword id mapping.')
+    args = parser.parse_args()
+
+    if not Path(args.infile).is_file():
+        raise FileNotFoundError(args.infile)
+
+    if not Path(args.bpe_model).is_file():
+        raise FileNotFoundError(args.bpe_model)
+
+    with open(args.infile) as f:
+        infile = f.readlines()
+    infile = [x.strip() for x in infile]
+
+    sp = spm.SentencePieceProcessor()
+    sp.load(args.bpe_model)
+
+    out_commands, tokens = process_commands(infile, sp)
+
+    token_symbols = []
+    for i in range(sp.vocab_size()):
+        if sp.id_to_piece(i) in tokens or i == 0:
+            token_symbols.append(f'{sp.id_to_piece(i)}\t{i}')
+
+    with open(args.out_command_list, 'wt') as f:
+        f.write('\n'.join(out_commands))
+        f.write('\n')
+
+    with open(args.out_token_symbols, 'wt') as f:
+        f.write('\n'.join(token_symbols))
+        f.write('\n')

managed_components/espressif__esp-sr/tool/fst/requirements.txt

@@ -0,0 +1 @@
+sentencepiece==0.1.97

managed_components/espressif__esp-sr/tool/multinet_g2p.py

@@ -0,0 +1,45 @@
+from g2p_en import G2p
+import argparse
+import numpy as np
+import pandas
+
+def english_g2p(text, alphabet=None):
+    g2p = G2p()
+    out = ""
+
+    if alphabet is None:
+        alphabet={"AE1": "a", "N": "N", " ": " ", "OW1": "b", "V": "V", "AH0": "c", "L": "L", "F": "F", "EY1": "d", "S": "S", "B": "B", "R": "R", "AO1": "e", "D": "D", "AH1": "c", "EH1": "f", "OW0": "b", "IH0": "g", "G": "G", "HH": "h", "K": "K", "IH1": "g", "W": "W", "AY1": "i", "T": "T", "M": "M", "Z": "Z", "DH": "j", "ER0": "k", "P": "P", "NG": "l", "IY1": "m", "AA1": "n", "Y": "Y", "UW1": "o", "IY0": "m", "EH2": "f", "CH": "p", "AE0": "a", "JH": "q", "ZH": "r", "AA2": "n", "SH": "s", "AW1": "t", "OY1": "u", "AW2": "t", "IH2": "g", "AE2": "a", "EY2": "d", "ER1": "k", "TH": "v", "UH1": "w", "UW2": "o", "OW2": "b", "AY2": "i", "UW0": "o", "AH2": "c", "EH0": "f", "AW0": "t", "AO2": "e", "AO0": "e", "UH0": "w", "UH2": "w", "AA0": "n", "AY0": "i", "IY2": "m", "EY0": "d", "ER2": "k", "OY2": "u", "OY0": "u"}
+
+    text_list = text.split(";")
+    for item in text_list:
+        item = item.split(",")
+        for phrase in item:
+            labels = g2p(phrase)
+            for char in labels:
+                if char not in alphabet:
+                    print("skip %s, not found in alphabet")
+                    continue
+                else:
+                    out += alphabet[char]
+            if phrase != item[-1]:
+                out += ','
+        out += ";"
+
+    print("in:", text)
+    print("out:", out)
+    
+    return out
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(prog="English Speech Commands G2P")
+
+    parser.add_argument("--text", "-t", type=str,  default=None, help="input text")
+    parser.add_argument("--alphabet_map", "-a", type=str,  default=None, help="the json file to map label into classes of model")
+    parser.add_argument('-c', '--c_file', help="name of .c files")
+    parser.add_argument('-head', '--h_file', help="name of .h files")
+    args = parser.parse_args()
+    
+    if args.text is not None:
+        english_g2p(args.text, args.alphabet_map)
+

managed_components/espressif__esp-sr/tool/multinet_pinyin.py

@@ -0,0 +1,47 @@
+from pypinyin import lazy_pinyin
+from pypinyin import load_phrases_dict
+from pypinyin import load_single_dict
+from pypinyin_dict.phrase_pinyin_data import large_pinyin
+import argparse
+
+
+phrases_dict = {
+            '开户行': [['ka1i'], ['hu4'], ['hang2']],
+            '发卡行': [['fa4'], ['ka3'], ['hang2']],
+            '放款行': [['fa4ng'], ['kua3n'], ['hang2']],
+            '茧行': [['jia3n'], ['hang2']],
+            '行号': [['hang2'], ['ha4o']],
+            '各地': [['ge4'], ['di4']],
+            '借还款': [['jie4'], ['hua2n'], ['kua3n']],
+            '时间为': [['shi2'], ['jia1n'], ['we2i']],
+            '为准': [['we2i'], ['zhu3n']],
+            '色差': [['se4'], ['cha1']],
+            '嗲': [['dia3']],
+            '呗': [['bei5']],
+            '不': [['bu4']],
+            '咗': [['zuo5']],
+            '嘞': [['lei5']],
+            '掺和': [['chan1'], ['huo5']]
+        }
+
+def init_pinyin():
+    large_pinyin.load()
+    load_phrases_dict(phrases_dict)
+    load_single_dict({22320: u'de,di4'})   # 地
+    load_single_dict({35843: u'tiao2,diao4'})   #调
+
+def get_pinyin(text):
+    label = lazy_pinyin(text)
+    label = " ".join(label)
+
+    print("in:", text)
+    print("out:", label)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="Chinese Speech Commands")
+    parser.add_argument("--text", "-t", type=str,  default=None, help="input text")
+    args = parser.parse_args()
+
+    init_pinyin()
+    get_pinyin(args.text)

managed_components/espressif__esp-sr/tool/requirements

@@ -0,0 +1,3 @@
+g2p-en
+pypinyin
+pypinyin_dict