Adding Speech Output to Cubetto

Owned by Simon Bates
Last updated: Oct 15, 2019
8 min read

Cubetto Hardware

Cubetto Firmware

Primo Video Guide to Updating the Cubetto Firmware Using the Arduino IDE:

Firmware Files

Speech Option 1: Text-To-Speech Library

Speech Option 2: Playback of Recorded Audio

Outline

  • Prepare a recording of the word that we want to have Cubetto speak
  • Store the audio data in our program by using 'xxd' to generate a C array notation for the audio data
  • Use the Arduino PROGMEM keyword to store the audio data in flash storage, rather than RAM

Preparing Audio using eSpeak NG, FFmpeg, and xxd

We can use the eSpeak NG text-to-speech system to generate speech samples. For example to make a sample file for the word "forward":

> espeak-ng -v en -w forward.wav "forward"

The audio data from eSpeak NG is 16-bit at 22050 Hz in WAV file format:

> ffprobe forward.wav
Input #0, wav, from 'forward.wav':
Duration: 00:00:00.81, bitrate: 353 kb/s
Stream #0:0: Audio: pcm_s16le ([1][0][0][0] / 0x0001), 22050 Hz, 1 channels, s16, 352 kb/s

Use FFmpeg to convert to 8-bit and strip the file header information:

> ffmpeg -i forward.wav -f u8 forward_u8_22050.raw

Make a C array notation version of the audio data with xxd:

> xxd -i forward_u8_22050.raw > forward_u8_22050.h

Change the array type to byte and add the PROGMEM keyword:

byte forward_u8_22050_raw[] PROGMEM = {
...

See: https://github.com/simonbates/arduino-audio-examples/blob/master/samd21_dac_sampled_audio/forward_u8_22050.h

Option 2A: Playback using the SAMD21 DAC

Proof of Concept Code for MKR1000

// Based on
// https://github.com/arduino-libraries/AudioZero/blob/master/src/AudioZero.cpp

#include "forward_u8_22050.h"

byte* audioData;
uint32_t audioDataLen;
uint32_t sampleIndex;

void TC5_Handler (void) __attribute__ ((weak, alias("Play_Sample_Handler")));

bool tcIsSyncing()
{
    return TC5->COUNT16.STATUS.reg & TC_STATUS_SYNCBUSY;
}

void resetTc()
{
    TC5->COUNT16.CTRLA.reg = TC_CTRLA_SWRST;
    while (tcIsSyncing());
    while (TC5->COUNT16.CTRLA.bit.SWRST);
}

void disableTc()
{
    TC5->COUNT16.CTRLA.reg &= ~TC_CTRLA_ENABLE;
    while (tcIsSyncing());
}

void configureTc(uint32_t sampleRate)
{
    // Enable GCLK for TCC2 and TC5 (timer counter input clock)
    GCLK->CLKCTRL.reg = (uint16_t) (GCLK_CLKCTRL_CLKEN | GCLK_CLKCTRL_GEN_GCLK0 | GCLK_CLKCTRL_ID(GCM_TC4_TC5)) ;
    while (GCLK->STATUS.bit.SYNCBUSY);

    resetTc();

    // Set Timer counter Mode to 16 bits
    TC5->COUNT16.CTRLA.reg |= TC_CTRLA_MODE_COUNT16;

    // Set TC5 mode as match frequency
    TC5->COUNT16.CTRLA.reg |= TC_CTRLA_WAVEGEN_MFRQ;

    TC5->COUNT16.CTRLA.reg |= TC_CTRLA_PRESCALER_DIV1 | TC_CTRLA_ENABLE;

    TC5->COUNT16.CC[0].reg = (uint16_t) (SystemCoreClock / sampleRate - 1);
    while (tcIsSyncing());
    
    // Configure interrupt request
    NVIC_DisableIRQ(TC5_IRQn);
    NVIC_ClearPendingIRQ(TC5_IRQn);
    NVIC_SetPriority(TC5_IRQn, 0);
    NVIC_EnableIRQ(TC5_IRQn);

    // Enable the TC5 interrupt request
    TC5->COUNT16.INTENSET.bit.MC0 = 1;
    while (tcIsSyncing());
}

void playSample(byte* data, uint32_t length, uint32_t sampleRate)
{
    pinMode(PIN_A0, OUTPUT);
    analogWriteResolution(8);

    audioData = data;
    audioDataLen = length;
    sampleIndex = 0;

    configureTc(sampleRate);
}

void stopPlaying()
{
    resetTc();
    disableTc();
}

void sayForward()
{
    playSample(forward_u8_22050_raw, forward_u8_22050_raw_len, 22050);
    delay(3000);
    stopPlaying();
}

void setup() {
}

void loop() {
    sayForward();
    delay(4000);
}

#ifdef __cplusplus
extern "C" {
#endif

void Play_Sample_Handler (void)
{
    if (sampleIndex < audioDataLen - 1) {
        analogWrite(PIN_A0, pgm_read_byte_far(audioData + sampleIndex));
        ++sampleIndex;

        // Clear the interrupt
        TC5->COUNT16.INTFLAG.bit.MC0 = 1;
    } else {
        // This will loop until stopPlaying() is called
        sampleIndex = 0;
        TC5->COUNT16.INTFLAG.bit.MC0 = 1;
    }
}

#ifdef __cplusplus
}
#endif

GitHub: https://github.com/simonbates/arduino-audio-examples/blob/master/samd21_dac_sampled_audio

Option 2B: Playback using PWM

Proof of Concept Code for MKR1000

// PWM based on code from MartinL in
// https://forum.arduino.cc/index.php?topic=346731.0
// Sample playback based on 
// https://github.com/arduino-libraries/AudioZero/blob/master/src/AudioZero.cpp

#include "forward_u8_22050.h"

byte* audioData;
uint32_t audioDataLen;
uint32_t sampleIndex;

void TC5_Handler (void) __attribute__ ((weak, alias("Play_Sample_Handler")));

bool tcIsSyncing()
{
    return TC5->COUNT16.STATUS.reg & TC_STATUS_SYNCBUSY;
}

void resetTc()
{
    TC5->COUNT16.CTRLA.reg = TC_CTRLA_SWRST;
    while (tcIsSyncing());
    while (TC5->COUNT16.CTRLA.bit.SWRST);
}

void disableTc()
{
    TC5->COUNT16.CTRLA.reg &= ~TC_CTRLA_ENABLE;
    while (tcIsSyncing());
}

void configureTc(uint32_t sampleRate)
{
    // Enable GCLK for TCC2 and TC5 (timer counter input clock)
    GCLK->CLKCTRL.reg = (uint16_t) (GCLK_CLKCTRL_CLKEN | GCLK_CLKCTRL_GEN_GCLK0 | GCLK_CLKCTRL_ID(GCM_TC4_TC5)) ;
    while (GCLK->STATUS.bit.SYNCBUSY);

    resetTc();

    // Set Timer counter Mode to 16 bits
    TC5->COUNT16.CTRLA.reg |= TC_CTRLA_MODE_COUNT16;

    // Set TC5 mode as match frequency
    TC5->COUNT16.CTRLA.reg |= TC_CTRLA_WAVEGEN_MFRQ;

    TC5->COUNT16.CTRLA.reg |= TC_CTRLA_PRESCALER_DIV1 | TC_CTRLA_ENABLE;

    TC5->COUNT16.CC[0].reg = (uint16_t) (SystemCoreClock / sampleRate - 1);
    while (tcIsSyncing());
    
    // Configure interrupt request
    NVIC_DisableIRQ(TC5_IRQn);
    NVIC_ClearPendingIRQ(TC5_IRQn);
    NVIC_SetPriority(TC5_IRQn, 0);
    NVIC_EnableIRQ(TC5_IRQn);

    // Enable the TC5 interrupt request
    TC5->COUNT16.INTENSET.bit.MC0 = 1;
    while (tcIsSyncing());
}

void playSample(byte* data, uint32_t length, uint32_t sampleRate)
{
    audioData = data;
    audioDataLen = length;
    sampleIndex = 0;

    configureTc(sampleRate);
}

void stopPlaying()
{
    resetTc();
    disableTc();
}

void sayForward()
{
    playSample(forward_u8_22050_raw, forward_u8_22050_raw_len, 22050);
    delay(3000);
    stopPlaying();
}

void setup()
{
    REG_GCLK_GENDIV = GCLK_GENDIV_DIV(1) |          // Divide the 48MHz clock source by divisor 1: 48MHz/1=48MHz
                      GCLK_GENDIV_ID(4);            // Select Generic Clock (GCLK) 4
    while (GCLK->STATUS.bit.SYNCBUSY);              // Wait for synchronization

    REG_GCLK_GENCTRL = GCLK_GENCTRL_IDC |           // Set the duty cycle to 50/50 HIGH/LOW
                       GCLK_GENCTRL_GENEN |         // Enable GCLK4
                       GCLK_GENCTRL_SRC_DFLL48M |   // Set the 48MHz clock source
                       GCLK_GENCTRL_ID(4);          // Select GCLK4
    while (GCLK->STATUS.bit.SYNCBUSY);              // Wait for synchronization

    // Enable the port multiplexer for the digital pin D7
    PORT->Group[g_APinDescription[7].ulPort].PINCFG[g_APinDescription[7].ulPin].bit.PMUXEN = 1;
 
    // Connect the TCC0 timer to digital output D7 - port pins are paired odd PMUO and even PMUXE
    // F & E specify the timers: TCC0, TCC1 and TCC2
    PORT->Group[g_APinDescription[6].ulPort].PMUX[g_APinDescription[6].ulPin >> 1].reg = PORT_PMUX_PMUXO_F;

    // Feed GCLK4 to TCC0 and TCC1
    REG_GCLK_CLKCTRL = GCLK_CLKCTRL_CLKEN |         // Enable GCLK4 to TCC0 and TCC1
                       GCLK_CLKCTRL_GEN_GCLK4 |     // Select GCLK4
                       GCLK_CLKCTRL_ID_TCC0_TCC1;   // Feed GCLK4 to TCC0 and TCC1
    while (GCLK->STATUS.bit.SYNCBUSY);              // Wait for synchronization

    // Dual slope PWM operation: timers countinuously count up to PER register value then down 0
    REG_TCC0_WAVE |= TCC_WAVE_POL(0xF) |           // Reverse the output polarity on all TCC0 outputs
                     TCC_WAVE_WAVEGEN_DSBOTH;      // Setup dual slope PWM on TCC0
    while (TCC0->SYNCBUSY.bit.WAVE);               // Wait for synchronization

    // Each timer counts up to a maximum or TOP value set by the PER register,
    // this determines the frequency of the PWM operation:
    REG_TCC0_PER = 255;
    while (TCC0->SYNCBUSY.bit.PER);                // Wait for synchronization
 
    REG_TCC0_CC3 = 0;         // TCC0 CC3 - on D7
    while (TCC0->SYNCBUSY.bit.CC3);                // Wait for synchronization
 
    // Divide the 48MHz signal by 1 giving 48MHz TCC0 timer tick and enable the outputs
    REG_TCC0_CTRLA |= TCC_CTRLA_PRESCALER_DIV1 |   // Divide GCLK4 by 1
                      TCC_CTRLA_ENABLE;            // Enable the TCC0 output
    while (TCC0->SYNCBUSY.bit.ENABLE);             // Wait for synchronization
}

void loop() {
    sayForward();
    delay(4000);
}

#ifdef __cplusplus
extern "C" {
#endif

void Play_Sample_Handler (void)
{
    if (sampleIndex < audioDataLen - 1) {
        // TCC0 CC3 - on D7
        REG_TCC0_CC3 = pgm_read_byte_far(audioData + sampleIndex);
        ++sampleIndex;

        // Clear the interrupt
        TC5->COUNT16.INTFLAG.bit.MC0 = 1;
    } else {
        // This will loop until stopPlaying() is called
        sampleIndex = 0;
        TC5->COUNT16.INTFLAG.bit.MC0 = 1;
    }
}

#ifdef __cplusplus
}
#endif

GitHub: https://github.com/simonbates/arduino-audio-examples/tree/master/samd21_pwm_sampled_audio

Resources

SAMD21 Product Information and Datasheet

Arduino Memory

Audio Libraries and Code Samples

Arduino/SAMD21 Timers