feat(voice): integrate Janis Joplin XTTS-v2 zero-shot voice cloning on MacMini M4
This commit is contained in:
parent
6c791c1618
commit
184cf44fdf
2 changed files with 37 additions and 10 deletions
|
|
@ -17,6 +17,7 @@ services:
|
||||||
- ./exports:/app/exports
|
- ./exports:/app/exports
|
||||||
- ./data:/app/data
|
- ./data:/app/data
|
||||||
- ./handlers:/app/handlers
|
- ./handlers:/app/handlers
|
||||||
|
- ./services:/app/services
|
||||||
- ../libs/swarmlib:/app/swarmlib:ro
|
- ../libs/swarmlib:/app/swarmlib:ro
|
||||||
- /home/matrixhasyou/domovoy_google_creds.json:/home/matrixhasyou/domovoy_google_creds.json:ro
|
- /home/matrixhasyou/domovoy_google_creds.json:/home/matrixhasyou/domovoy_google_creds.json:ro
|
||||||
- /home/matrixhasyou/mutt:/home/matrixhasyou/mutt:rw
|
- /home/matrixhasyou/mutt:/home/matrixhasyou/mutt:rw
|
||||||
|
|
@ -45,6 +46,7 @@ services:
|
||||||
- ./exports:/app/exports
|
- ./exports:/app/exports
|
||||||
- ./data:/app/data
|
- ./data:/app/data
|
||||||
- ./handlers:/app/handlers
|
- ./handlers:/app/handlers
|
||||||
|
- ./services:/app/services
|
||||||
- ../libs/swarmlib:/app/swarmlib:ro
|
- ../libs/swarmlib:/app/swarmlib:ro
|
||||||
- /home/matrixhasyou/domovoy_google_creds.json:/home/matrixhasyou/domovoy_google_creds.json:ro
|
- /home/matrixhasyou/domovoy_google_creds.json:/home/matrixhasyou/domovoy_google_creds.json:ro
|
||||||
- /home/matrixhasyou/mutt:/home/matrixhasyou/mutt:rw
|
- /home/matrixhasyou/mutt:/home/matrixhasyou/mutt:rw
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ from aiogram.types import Message, BufferedInputFile
|
||||||
from config import ADMIN_USER_ID, DATABASE_PATH
|
from config import ADMIN_USER_ID, DATABASE_PATH
|
||||||
import google.generativeai as genai
|
import google.generativeai as genai
|
||||||
import edge_tts
|
import edge_tts
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
router = Router()
|
router = Router()
|
||||||
|
|
@ -113,7 +114,7 @@ def log_voice_to_db(user_id, username, msg_type, duration, text):
|
||||||
logger.error(f"❌ DB Log error: {e}")
|
logger.error(f"❌ DB Log error: {e}")
|
||||||
|
|
||||||
async def text_to_speech_bytes(text: str) -> bytes:
|
async def text_to_speech_bytes(text: str) -> bytes:
|
||||||
"""Синтез качественного женского голосового ответа на русском через Edge TTS с глубоким тембром"""
|
"""Синтез голоса Дженис Джоплин через локальный XTTS-v2 на MacMini M4 (с фоллбэком на Edge TTS)"""
|
||||||
match = re.search(r"<voice>(.*?)</voice>", text, re.DOTALL | re.IGNORECASE)
|
match = re.search(r"<voice>(.*?)</voice>", text, re.DOTALL | re.IGNORECASE)
|
||||||
if match:
|
if match:
|
||||||
speech_text = match.group(1).strip()
|
speech_text = match.group(1).strip()
|
||||||
|
|
@ -128,6 +129,27 @@ async def text_to_speech_bytes(text: str) -> bytes:
|
||||||
if not speech_text:
|
if not speech_text:
|
||||||
speech_text = "Приняла, Александр. Всё проанализировано и готово."
|
speech_text = "Приняла, Александр. Всё проанализировано и готово."
|
||||||
|
|
||||||
|
# 1. Приоритет: локальный синтез голосом Дженис Джоплин на MacMini M4 (XTTS-v2 Apple Metal)
|
||||||
|
try:
|
||||||
|
timeout = aiohttp.ClientTimeout(total=20)
|
||||||
|
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||||
|
payload = {
|
||||||
|
"text": speech_text,
|
||||||
|
"language": "ru",
|
||||||
|
"speaker_wav": "/Users/matrixhasyou/janis_ref_15s.wav"
|
||||||
|
}
|
||||||
|
async with session.post("http://192.168.10.167:8050/tts", json=payload) as resp:
|
||||||
|
if resp.status == 200:
|
||||||
|
data = await resp.read()
|
||||||
|
if len(data) > 1000:
|
||||||
|
logger.info(f"🎤 Голос Дженис Джоплин (XTTS-v2 MacMini) успешно синтезирован ({len(data)} байт)")
|
||||||
|
return data
|
||||||
|
logger.warning(f"XTTS вернул HTTP {resp.status}, переход на Edge TTS")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"XTTS на MacMini недоступен ({e}), переход на Edge TTS")
|
||||||
|
|
||||||
|
# 2. Фоллбэк: Edge TTS (ru-RU-SvetlanaNeural)
|
||||||
|
try:
|
||||||
communicate = edge_tts.Communicate(speech_text, "ru-RU-SvetlanaNeural", pitch="-8Hz", rate="-3%")
|
communicate = edge_tts.Communicate(speech_text, "ru-RU-SvetlanaNeural", pitch="-8Hz", rate="-3%")
|
||||||
out_path = f"/tmp/tts_{os.getpid()}_{int(datetime.now().timestamp())}.mp3"
|
out_path = f"/tmp/tts_{os.getpid()}_{int(datetime.now().timestamp())}.mp3"
|
||||||
await communicate.save(out_path)
|
await communicate.save(out_path)
|
||||||
|
|
@ -137,6 +159,9 @@ async def text_to_speech_bytes(text: str) -> bytes:
|
||||||
audio_bytes = f.read()
|
audio_bytes = f.read()
|
||||||
os.remove(out_path)
|
os.remove(out_path)
|
||||||
return audio_bytes
|
return audio_bytes
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Ошибка Edge TTS: {e}")
|
||||||
|
return b""
|
||||||
|
|
||||||
def process_audio_with_gemini(audio_path: str) -> str:
|
def process_audio_with_gemini(audio_path: str) -> str:
|
||||||
"""Прямое транскрибирование аудио через Gemini Multimodal Audio API"""
|
"""Прямое транскрибирование аудио через Gemini Multimodal Audio API"""
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue