"""
Module de normalisation et d'assainissement textuel pour STT, LLM et TTS.
Fournit des fonctions pures et rapides pour eliminer les emojis, normaliser
la ponctuation, supprimer le balisage XML et preparer le texte a la synthese vocale.
"""
from __future__ import annotations
import re
from typing import ClassVar
[docs]
class TextCleaner:
"""
Utilitaire de nettoyage et de normalisation textuelle pour STT, LLM et TTS.
"""
noisePhrases: ClassVar[set[str]] = {
"...",
"..",
".",
"Merci.",
"Sous-titres réalisés par",
"Sous-titrage",
"Sous-titres",
"Transcrit par",
"DD",
"dd",
"D.D.",
"d.d.",
"Euh",
"euh",
}
EMOJI_PATTERN = re.compile(
r"[\U00010000-\U0010ffff\u2600-\u26ff\u2700-\u27bf\ufe00-\ufe0f\u200d]+",
flags=re.UNICODE,
)
_RE_THINKING = re.compile(r"<(?:thinking|thought|think)>.*?</(?:thinking|thought|think)>", flags=re.DOTALL | re.IGNORECASE)
_RE_CHANNEL_THOUGHT = re.compile(r"<\|channel>thought.*?<channel\|>", flags=re.DOTALL | re.IGNORECASE)
_RE_THOUGHT_LINE = re.compile(r"(?:^|\n)thought\n.*?(?=\n\n[A-ZÀ-Ö])", flags=re.DOTALL | re.IGNORECASE)
_RE_CHANNEL_TAGS = re.compile(r"<\|(?:turn>model|channel>text|channel>thought)>?\n?", flags=re.IGNORECASE)
_RE_CHANNEL_CLOSE = re.compile(r"<channel\|>", flags=re.IGNORECASE)
_RE_XML_ALL = re.compile(r"<[^>]+>")
_RE_SOURCES = re.compile(r"\[?\bSource\s*\d+\b\]?\s*:?", flags=re.IGNORECASE)
_RE_LEADING_DASH = re.compile(r"^[\s\-]+", flags=re.MULTILINE)
_RE_CONJUNCTIONS = re.compile(r",\s*(et|ou|mais|donc|car|que|qui|avec)\b", flags=re.IGNORECASE)
_RE_REPEATED_COMMAS = re.compile(r",\s*,+")
_RE_DOTS = re.compile(r"\s*\.{2,}\s*")
_RE_SPACES = re.compile(r"[ \t]+")
_RE_MULTI_SPACES = re.compile(r"\s+")
_RE_STT_WORD_REPETITION = re.compile(r"\b(?!(?:nous|vous)\b)(\w+)(?:\s+\1\b)+", flags=re.IGNORECASE)
_RE_STT_REPEATED_PUNCT = re.compile(r"([.?!,;:])\s*\1+")
_MARKDOWN_TRANSLATION = str.maketrans("", "", "*_`#~>[](){}|–—") # noqa: RUF001
[docs]
@classmethod
def stripEmojis(cls, text: str) -> str:
"""
Supprime tous les emojis et pictogrammes d'une chaine de caracteres.
Parameters
----------
text : str
Chaine source potentiellement porteuse d'emojis.
Returns
-------
str
Chaine assainie depourvue d'emojis.
"""
if not text:
return ""
cleaned = cls.EMOJI_PATTERN.sub("", text)
return cls._RE_SPACES.sub(" ", cleaned)
[docs]
@classmethod
def cleanSttText(cls, text: str) -> str:
"""
Nettoie et assainit le texte transcrit par le moteur STT.
Deduplonne les begaiements acoustiques de l'ASR (ex: 'de de', 'propos
propos') sans toucher aux formes pronominales ('nous nous', 'vous vous'),
puis normalise espaces et ponctuations multiples.
Parameters
----------
text : str
Texte brut issu du moteur STT.
Returns
-------
str
Texte nettoye, sans begaiements ni artefacts de ponctuation.
"""
if not text:
return ""
# Espaces multiples
cleaned = cls._RE_MULTI_SPACES.sub(" ", text).strip()
# Repetitions consecutives de mots identiques
cleaned = cls._RE_STT_WORD_REPETITION.sub(r"\1", cleaned)
# Ponctuation redondante (ex: '??', '..')
cleaned = cls._RE_STT_REPEATED_PUNCT.sub(r"\1", cleaned)
# Espaces avant la ponctuation faible
cleaned = re.sub(r"\s+([,.;])", r"\1", cleaned)
return cleaned.strip()
[docs]
@classmethod
def isNoiseTranscription(cls, text: str) -> bool:
"""
Detecte si une transcription ne contient que des artefacts ou du bruit.
Parameters
----------
text : str
Texte issu du moteur STT.
Returns
-------
bool
True si le texte est du bruit sans valeur semantique, False sinon.
"""
trimmed = text.strip()
if len(trimmed) < 2:
return True
if trimmed in cls.noisePhrases:
return True
# Lettres isolees repetees (ex: 'D D', 'd d')
if re.fullmatch(r"([A-Za-z])(?:\s+\1)+", trimmed, flags=re.IGNORECASE):
return True
return False
[docs]
@classmethod
def cleanForTts(cls, rawText: str) -> str:
"""
Nettoie le texte genere pour l'optimiser pour la synthese vocale TTS.
Parameters
----------
rawText : str
Texte brut genere par le LLM.
Returns
-------
str
Texte nettoye, sans balises ni markdown, optimise pour la diction orale.
"""
if not rawText:
return ""
cleaned = cls.stripEmojis(rawText)
cleaned = cls._RE_THINKING.sub("", cleaned)
cleaned = cls._RE_CHANNEL_THOUGHT.sub("", cleaned)
cleaned = cls._RE_THOUGHT_LINE.sub("", cleaned)
cleaned = cls._RE_CHANNEL_TAGS.sub("", cleaned)
cleaned = cls._RE_CHANNEL_CLOSE.sub("", cleaned)
cleaned = cls._RE_XML_ALL.sub("", cleaned)
cleaned = cls._RE_SOURCES.sub("", cleaned)
cleaned = cleaned.translate(cls._MARKDOWN_TRANSLATION)
cleaned = cls._RE_LEADING_DASH.sub("", cleaned)
cleaned = cls._RE_CONJUNCTIONS.sub(r" \1", cleaned)
cleaned = cls._RE_REPEATED_COMMAS.sub(",", cleaned)
cleaned = cls._RE_DOTS.sub(". ", cleaned)
cleaned = cls._RE_MULTI_SPACES.sub(" ", cleaned)
return cleaned.strip()