Source code for annotations.Phon.phonetize

# -*- coding: UTF-8 -*-
:author:   Brigitte Bigi
:summary:  Phonetization of an utterance.

.. _This file is part of SPPAS:

     ___   __    __    __    ___
    /     |  \  |  \  |  \  /              the automatic
    \__   |__/  |__/  |___| \__             annotation and
       \  |     |     |   |    \             analysis
    ___/  |     |     |   | ___/              of speech

    Copyright (C) 2011-2021  Brigitte Bigi
    Laboratoire Parole et Langage, Aix-en-Provence, France

    Use of this software is governed by the GNU Public License, version 3.

    SPPAS is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    SPPAS is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with SPPAS. If not, see <>.

    This banner notice must not be removed.



import re

from sppas.src.config import symbols
from sppas.src.config import separators
from sppas.src.config import annots
from sppas.src.config import sppasUnicode
from sppas.src.config import u
from sppas.src.resources import sppasMapping
from sppas.src.resources import sppasDictPron

from .phonunk import sppasPhonUnk
from .dagphon import sppasDAGPhonetizer

# ---------------------------------------------------------------------------

SIL = list([list("silence")]

# ---------------------------------------------------------------------------

[docs]class sppasDictPhonetizer(object): """Dictionary-based automatic phonetization. Grapheme-to-phoneme conversion is a complex task, for which a number of diverse solutions have been proposed. It is a structure prediction task; both the input and output are structured, consisting of sequences of letters and phonemes, respectively. This phonetization system is entirely designed to handle multiple languages and/or tasks with the same algorithms and the same tools. Only resources are language-specific, and the approach is based on the simplest resources as possible: this automatic annotation is using a dictionary-based approach. The dictionary can contain words with a set of pronunciations (the canonical one, and optionally some common reductions, etc). In this approach, it is then assumed that most of the words of the speech transcription and their phonetic variants are mentioned in the pronunciation dictionary. If a word is missing, our system is based on the idea that given enough examples it should be possible to predict the pronunciation of unseen words purely by analogy. """
[docs] def __init__(self, pdict, maptable=None): """Create a sppasDictPhonetizer instance. :param pdict: (sppasDictPron) The pronunciation dictionary. :param maptable: (Mapping) A mapping table for phones. """ self._pdict = None self._phonunk = None self._map_table = sppasMapping() self._dag_phon = sppasDAGPhonetizer() self.set_dict(pdict) self.set_maptable(maptable)
# -----------------------------------------------------------------------
[docs] def get_dict_filename(self): if self._pdict is None: return "" return self._pdict.get_filename()
# -----------------------------------------------------------------------
[docs] def set_dict(self, pron_dict): """Set the pronunciation dictionary. :param pron_dict: (sppasDictPron) The pronunciation dictionary. """ if pron_dict is None: pron_dict = sppasDictPron() if isinstance(pron_dict, sppasDictPron) is False: raise TypeError('Expected a sppasDictPron instance.') self._pdict = pron_dict self._phonunk = sppasPhonUnk(self._pdict)
# -----------------------------------------------------------------------
[docs] def set_maptable(self, map_table): """Set the mapping table dictionary. :param map_table: (Mapping) The mapping table dictionary. """ if map_table is not None: if isinstance(map_table, sppasMapping) is False: raise TypeError('Expected a Mapping instance.') else: map_table = sppasMapping() self._map_table = map_table self._map_table.set_keep_miss(False)
# -----------------------------------------------------------------------
[docs] def set_unk_variants(self, value): """Fix the maximum number of variants for unknown entries. :param value: (int) If v is set to 0, all variants will be returned. """ self._dag_phon.set_variants(value)
# -----------------------------------------------------------------------
[docs] def get_phon_entry(self, entry): """Return the phonetization of an entry. Unknown entries are not automatically phonetized. This is a pure dictionary-based method. :param entry: (str) The entry to be phonetized. :returns: A string with the phonetization of the given entry or the unknown symbol. """ entry = sppasUnicode(entry).to_strip() # Specific strings... for the italian transcription... # For the participation at the CLIPS-Evalita 2011 campaign. if entry.startswith(u("<")) is True and entry.endswith(u(">")) is True: entry = entry[1:-1] # No entry! Nothing to do. if len(entry) == 0: return "" # Specific strings used in the CID transcriptions... # CID is Corpus of Interactionnal Data, if entry.startswith(u("gpf_")) is True: return SIL if entry.startswith(u("gpd_")) is True: return "" # Specific strings used in SPPAS IPU segmentation... if entry.startswith(u("ipu_")): return "" # Find entry in the dict as it is given _strphon = self._pdict.get_pron(entry) # OK, the entry is properly phonetized. if _strphon != self._pdict.get_unkstamp(): return self._map_phonentry(_strphon) return self._pdict.get_unkstamp()
# -----------------------------------------------------------------------
[docs] def get_phon_tokens(self, tokens, phonunk=True): """Return the phonetization of a list of tokens, with the status. Unknown entries are automatically phonetized if `phonunk` is set to True. :param tokens: (list) The list of tokens to be phonetized. :param phonunk: (bool) Phonetize unknown words (or not). TODO: EOT is not fully supported. :returns: A list with the tuple (token, phon, status). """ tab = list() for entry in tokens: entry = entry.strip() phon = self._pdict.get_unkstamp() status = annots.ok # Enriched Orthographic Transcription Convention: # entry can be already in SAMPA. if entry.startswith("/") is True and entry.endswith("/") is True: phon = entry.strip("/") # It must use X-SAMPA, # including minus character to separate phonemes. else: phon = self.get_phon_entry(entry) if phon == self._pdict.get_unkstamp(): status = annots.error # A missing compound word? if "-" in entry or "'" in entry or "_" in entry: _tabpron = [self.get_phon_entry(w) for w in re.split("[-'_]", entry)] # OK, finally the entry is in the dictionary? if self._pdict.get_unkstamp() not in _tabpron: # ATTENTION: each part can have variants! # must be decomposed. self._dag_phon.variants = 4 phon = sppasUnicode( self._dag_phon.decompose(" ".join(_tabpron))).to_strip() status = annots.warning if phon == self._pdict.get_unkstamp() and phonunk is True: try: phon = self._phonunk.get_phon(entry) status = annots.warning except: phon = self._pdict.get_unkstamp() status = annots.error if len(phon) > 0: tab.append((entry, phon, status)) return tab
# -----------------------------------------------------------------------
[docs] def phonetize(self, utterance, phonunk=True, delimiter=" "): """Return the phonetization of an utterance. :param utterance: (str) The utterance string to be phonetized. :param phonunk: (bool) Phonetize unknown words (or not). :param delimiter: (char) The character to be used to separate entries in the result and which was used in the given utterance. :returns: A string with the phonetization of the given utterance. """ if len(delimiter) > 1: raise TypeError('Delimiter must be a character.') su = sppasUnicode(utterance) utt = su.to_strip() tab = self.get_phon_tokens(utt.split(delimiter), phonunk) tab_phon = [t[1] for t in tab] phonetization = delimiter.join(tab_phon) return phonetization.strip()
# ----------------------------------------------------------------------- # Private # ----------------------------------------------------------------------- def _map_phonentry(self, phonentry): """Map phonemes of a phonetized entry. :param phonentry: (str) Phonetization of an entry. """ if self._map_table.is_empty() is True: return phonentry tab = [self._map_variant(v) for v in phonentry.split(separators.variants)] return separators.variants.join(tab) # ----------------------------------------------------------------------- def _map_variant(self, phonvariant): """Map phonemes of only one variant of a phonetized entry. :param phonvariant: (str) One phonetization variant of an entry. """ phones = self._map_split_variant(phonvariant) subs = [] # Single phonemes for p in phones: mapped = self._map_table.map_entry(p) if len(mapped) > 0: subs.append(p + separators.variants + mapped) else: subs.append(p) self._dag_phon.variants = 0 phon = sppasUnicode( self._dag_phon.decompose(" ".join(subs))).to_strip() # Remove un-pronounced phonemes!!! # By convention, they are represented by an underscore in the # mapping table. tmp = [] for p in phon.split(separators.variants): r = [x for x in p.split(separators.phonemes) if x != "_"] tmp.append(separators.phonemes.join(r)) return separators.variants.join(set(tmp)) # ----------------------------------------------------------------------- def _map_split_variant(self, phon_variant): """Return a list of the longest phone sequences. :param phon_variant: (str) One phonetization variant of an entry. """ phones = phon_variant.split(separators.phonemes) if len(phones) == 1: return phones tab = list() idx = 0 maxidx = len(phones) while idx < maxidx: # Find the index of the longest phone sequence that can be mapped left_index = self.__longestlr(phones[idx:maxidx]) # Append such a longest sequence in tab s = separators.phonemes tab.append(s.join(phones[idx:idx+left_index])) idx += left_index return tab # ----------------------------------------------------------------------- def __longestlr(self, tabentry): """Select the longest map of an entry.""" i = len(tabentry) while i > 0: # Find in the map table a substring from 0 to i entry = separators.phonemes.join(tabentry[:i]) if self._map_table.is_key(entry): return i i -= 1 # Did not find any map for this entry! Return the shortest. return 1