add some code

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/collect_multi_run_results.py

@@ -0,0 +1,190 @@
+"""
+/* Copyright (c) 2023 Amazon
+   Written by Jan Buethe */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+"""
+
+import argparse
+import os
+from uuid import UUID
+from collections import OrderedDict
+import pickle
+
+
+import torch
+import numpy as np
+
+import utils
+
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("input", type=str, help="input folder containing multi-run output")
+parser.add_argument("tag", type=str, help="tag for multi-run experiment")
+parser.add_argument("csv", type=str, help="name for output csv")
+
+
+def is_uuid(val):
+    try:
+        UUID(val)
+        return True
+    except:
+        return False
+
+
+def collect_results(folder):
+
+    training_folder = os.path.join(folder, 'training')
+    testing_folder  = os.path.join(folder, 'testing')
+
+    # validation loss
+    checkpoint = torch.load(os.path.join(training_folder, 'checkpoints', 'checkpoint_finalize_epoch_1.pth'), map_location='cpu')
+    validation_loss = checkpoint['validation_loss']
+
+    # eval_warpq
+    eval_warpq = utils.data.parse_warpq_scores(os.path.join(training_folder, 'out_finalize.txt'))[-1]
+
+    # testing results
+    testing_results = utils.data.collect_test_stats(os.path.join(testing_folder, 'final'))
+
+    results = OrderedDict()
+    results['eval_loss']          = validation_loss
+    results['eval_warpq']         = eval_warpq
+    results['pesq_mean']          = testing_results['pesq'][0]
+    results['warpq_mean']         = testing_results['warpq'][0]
+    results['pitch_error_mean']   = testing_results['pitch_error'][0]
+    results['voicing_error_mean'] = testing_results['voicing_error'][0]
+
+    return results
+
+def print_csv(path, results, tag, ranks=None, header=True):
+
+    metrics = next(iter(results.values())).keys()
+    if ranks is not None:
+        rank_keys = next(iter(ranks.values())).keys()
+    else:
+        rank_keys = []
+
+    with open(path, 'w') as f:
+        if header:
+            f.write("uuid, tag")
+
+            for metric in metrics:
+                f.write(f", {metric}")
+
+            for rank in rank_keys:
+                f.write(f", {rank}")
+
+            f.write("\n")
+
+
+        for uuid, values in results.items():
+            f.write(f"{uuid}, {tag}")
+
+            for val in values.values():
+                f.write(f", {val:10.8f}")
+
+            for rank in rank_keys:
+                f.write(f", {ranks[uuid][rank]:4d}")
+
+            f.write("\n")
+
+def get_ranks(results):
+
+    metrics = list(next(iter(results.values())).keys())
+
+    positive = {'pesq_mean', 'mix'}
+
+    ranks = OrderedDict()
+    for key in results.keys():
+        ranks[key] = OrderedDict()
+
+    for metric in metrics:
+        sign = -1 if metric in positive else 1
+
+        x = sorted([(key, value[metric]) for key, value in results.items()], key=lambda x: sign * x[1])
+        x = [y[0] for y in x]
+
+        for key in results.keys():
+            ranks[key]['rank_' + metric] = x.index(key) + 1
+
+    return ranks
+
+def analyse_metrics(results):
+    metrics = ['eval_loss', 'pesq_mean', 'warpq_mean', 'pitch_error_mean', 'voicing_error_mean']
+
+    x = []
+    for metric in metrics:
+        x.append([val[metric] for val in results.values()])
+
+    x = np.array(x)
+
+    print(x)
+
+def add_mix_metric(results):
+    metrics = ['eval_loss', 'pesq_mean', 'warpq_mean', 'pitch_error_mean', 'voicing_error_mean']
+
+    x = []
+    for metric in metrics:
+        x.append([val[metric] for val in results.values()])
+
+    x = np.array(x).transpose() * np.array([-1, 1, -1, -1, -1])
+
+    z = (x - np.mean(x, axis=0)) / np.std(x, axis=0)
+
+    print(f"covariance matrix for normalized scores of {metrics}:")
+    print(np.cov(z.transpose()))
+
+    score = np.mean(z, axis=1)
+
+    for i, key in enumerate(results.keys()):
+        results[key]['mix'] = score[i].item()
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    uuids = sorted([x for x in os.listdir(args.input) if os.path.isdir(os.path.join(args.input, x)) and is_uuid(x)])
+
+
+    results = OrderedDict()
+
+    for uuid in uuids:
+        results[uuid] = collect_results(os.path.join(args.input, uuid))
+
+
+    add_mix_metric(results)
+
+    ranks = get_ranks(results)
+
+
+
+    csv = args.csv if args.csv.endswith('.csv') else args.csv + '.csv'
+
+    print_csv(args.csv, results, args.tag, ranks=ranks)
+
+
+    with open(csv[:-4] + '.pickle', 'wb') as f:
+        pickle.dump(results, f, protocol=pickle.HIGHEST_PROTOCOL)

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/loop_run.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+
+case $# in
+    9) SETUP=$1; OUTDIR=$2; NAME=$3; DEVICE=$4; ROUNDS=$5; LPCNEXT=$6; LPCNET=$7; TESTSUITE=$8; TESTITEMS=$9;;
+    *) echo "loop_run.sh setup outdir name device rounds lpcnext_repo lpcnet_repo testsuite_repo testitems"; exit;;
+esac
+
+
+PYTHON="/home/ubuntu/opt/miniconda3/envs/torch/bin/python"
+TESTFEATURES=${LPCNEXT}/testitems/features/all_0_orig_features.f32
+WARPQREFERENCE=${LPCNEXT}/testitems/wav/all_0_orig.wav
+METRICS="warpq,pesq,pitch_error,voicing_error"
+LPCNETDEMO=${LPCNET}/lpcnet_demo
+
+for ((round = 1; round <= $ROUNDS; round++))
+do
+    echo
+    echo round $round
+
+    UUID=$(uuidgen)
+    TRAINOUT=${OUTDIR}/${UUID}/training
+    TESTOUT=${OUTDIR}/${UUID}/testing
+    CHECKPOINT=${TRAINOUT}/checkpoints/checkpoint_last.pth
+    FINALCHECKPOINT=${TRAINOUT}/checkpoints/checkpoint_finalize_last.pth
+
+    # run training
+    echo "starting training..."
+    $PYTHON $LPCNEXT/train_lpcnet.py $SETUP $TRAINOUT --device $DEVICE --test-features $TESTFEATURES --warpq-reference $WARPQREFERENCE
+
+    # run finalization
+    echo "starting finalization..."
+    $PYTHON $LPCNEXT/train_lpcnet.py $SETUP $TRAINOUT \
+            --device $DEVICE --test-features $TESTFEATURES \
+            --warpq-reference $WARPQREFERENCE \
+            --finalize --initial-checkpoint $CHECKPOINT
+
+    # create test configs
+    $PYTHON $LPCNEXT/make_test_config.py ${OUTDIR}/${UUID}/testconfig.yml "$NAME $UUID" $CHECKPOINT --lpcnet-demo $LPCNETDEMO
+    $PYTHON $LPCNEXT/make_test_config.py ${OUTDIR}/${UUID}/testconfig_finalize.yml "$NAME $UUID finalized" $FINALCHECKPOINT --lpcnet-demo $LPCNETDEMO
+
+    # run tests
+    echo "starting test 1 (no finalization)..."
+    $PYTHON $TESTSUITE/run_test.py ${OUTDIR}/${UUID}/testconfig.yml  \
+            $TESTITEMS ${TESTOUT}/prefinal --num-workers 8 \
+            --num-testitems 400 --metrics $METRICS
+
+    echo "starting test 2 (after finalization)..."
+    $PYTHON $TESTSUITE/run_test.py ${OUTDIR}/${UUID}/testconfig_finalize.yml  \
+            $TESTITEMS ${TESTOUT}/final --num-workers 8 \
+            --num-testitems 400 --metrics $METRICS
+done

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/make_animation.py

@@ -0,0 +1,67 @@
+"""
+/* Copyright (c) 2023 Amazon
+   Written by Jan Buethe */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+"""
+
+
+""" script for creating animations from debug data
+
+"""
+
+
+import argparse
+
+
+import sys
+sys.path.append('./')
+
+from utils.endoscopy import make_animation, read_data
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('folder', type=str, help='endoscopy folder with debug output')
+parser.add_argument('output', type=str, help='output file (will be auto-extended with .mp4)')
+
+parser.add_argument('--start-index', type=int, help='index of first sample to be considered', default=0)
+parser.add_argument('--stop-index', type=int, help='index of last sample to be considered', default=-1)
+parser.add_argument('--interval', type=int, help='interval between frames in ms', default=20)
+parser.add_argument('--half-window-length', type=int, help='half size of window for displaying signals', default=80)
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    filename = args.output if args.output.endswith('.mp4') else args.output + '.mp4'
+    data = read_data(args.folder)
+
+    make_animation(
+        data,
+        filename,
+        start_index=args.start_index,
+        stop_index = args.stop_index,
+        half_signal_window_length=args.half_window_length
+    )

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/modify_dataset_target.py

@@ -0,0 +1,17 @@
+import argparse
+
+import numpy as np
+
+
+parser = argparse.ArgumentParser(description="sets s_t to augmented_s_t")
+
+parser.add_argument('datafile', type=str, help='data.s16 file path')
+
+args = parser.parse_args()
+
+data = np.memmap(args.datafile, dtype='int16', mode='readwrite')
+
+# signal is in data[1::2]
+# last augmented signal is in data[0::2]
+
+data[1 : - 1 : 2] = data[2 : : 2]

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/multi_run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+case $# in
+    9) SETUP=$1; OUTDIR=$2; NAME=$3; NUMDEVICES=$4; ROUNDS=$5; LPCNEXT=$6; LPCNET=$7; TESTSUITE=$8; TESTITEMS=$9;;
+    *) echo "multi_run.sh setup outdir name num_devices rounds_per_device lpcnext_repo lpcnet_repo testsuite_repo testitems"; exit;;
+esac
+
+
+LOOPRUN=${LPCNEXT}/loop_run.sh
+
+mkdir -p $OUTDIR
+
+for ((i = 0; i < $NUMDEVICES; i++))
+do
+    echo "launching job queue for device $i"
+    nohup bash $LOOPRUN $SETUP $OUTDIR "$NAME" "cuda:$i" $ROUNDS $LPCNEXT $LPCNET $TESTSUITE $TESTITEMS > $OUTDIR/job_${i}_out.txt &
+done

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/run_inference_test.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+
+case $# in
+    3) FEATURES=$1; FOLDER=$2; PYTHON=$3;;
+    *) echo "run_inference_test.sh <features file> <output folder> <python path>"; exit;;
+esac
+
+
+SCRIPTFOLDER=$(dirname "$0")
+
+mkdir -p $FOLDER/inference_test
+
+# update checkpoints
+for fn in $(find $FOLDER -type f -name "checkpoint*.pth")
+do
+    tmp=$(basename $fn)
+    tmp=${tmp%.pth}
+    epoch=${tmp#checkpoint_epoch_}
+    echo "running inference with checkpoint $fn..."
+    $PYTHON $SCRIPTFOLDER/../test_lpcnet.py $FEATURES $fn $FOLDER/inference_test/output_epoch_${epoch}.wav
+done

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/update_checkpoints.py

@@ -0,0 +1,54 @@
+"""
+/* Copyright (c) 2023 Amazon
+   Written by Jan Buethe */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+"""
+
+""" script for updating checkpoints with new setup entries
+
+    Use this script to update older outputs with newly introduced
+    parameters. (Saves us the trouble of backward compatibility)
+"""
+
+
+import argparse
+
+import torch
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('checkpoint_file', type=str, help='checkpoint to be updated')
+parser.add_argument('--model', type=str, help='model update', default=None)
+
+args = parser.parse_args()
+
+checkpoint = torch.load(args.checkpoint_file, map_location='cpu')
+
+# update model entry
+if type(args.model) != type(None):
+    checkpoint['setup']['lpcnet']['model'] = args.model
+
+torch.save(checkpoint, args.checkpoint_file)

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/update_output_folder.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+
+case $# in
+    3) FOLDER=$1; MODEL=$2; PYTHON=$3;;
+    *) echo "update_output_folder.sh folder model python"; exit;;
+esac
+
+
+SCRIPTFOLDER=$(dirname "$0")
+
+
+# update setup
+echo "updating $FOLDER/setup.py..."
+$PYTHON $SCRIPTFOLDER/update_setups.py $FOLDER/setup.yml --model $MODEL
+
+# update checkpoints
+for fn in $(find $FOLDER -type f -name "checkpoint*.pth")
+do
+    echo "updating $fn..."
+    $PYTHON $SCRIPTFOLDER/update_checkpoints.py $fn --model $MODEL
+done

managed_components/78__esp-opus/dnn/torch/lpcnet/scripts/update_setups.py

@@ -0,0 +1,57 @@
+"""
+/* Copyright (c) 2023 Amazon
+   Written by Jan Buethe */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+"""
+
+""" script for updating setup files with new setup entries
+
+    Use this script to update older outputs with newly introduced
+    parameters. (Saves us the trouble of backward compatibility)
+"""
+
+import argparse
+
+import yaml
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('setup_file', type=str, help='setup to be updated')
+parser.add_argument('--model', type=str, help='model update', default=None)
+
+args = parser.parse_args()
+
+# load setup
+with open(args.setup_file, 'r') as f:
+    setup = yaml.load(f.read(), yaml.FullLoader)
+
+# update model entry
+if type(args.model) != type(None):
+    setup['lpcnet']['model'] = args.model
+
+# dump result
+with open(args.setup_file, 'w') as f:
+    yaml.dump(setup, f)