chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1 @@
|
||||
# Core base package
|
||||
@@ -0,0 +1,6 @@
|
||||
from core.base.tts.app_generator_tts_publisher import AppGeneratorTTSPublisher, AudioTrunk
|
||||
|
||||
__all__ = [
|
||||
"AppGeneratorTTSPublisher",
|
||||
"AudioTrunk",
|
||||
]
|
||||
@@ -0,0 +1,175 @@
|
||||
import base64
|
||||
import logging
|
||||
import queue
|
||||
import re
|
||||
import threading
|
||||
|
||||
from core.app.entities.queue_entities import (
|
||||
MessageQueueMessage,
|
||||
QueueAgentMessageEvent,
|
||||
QueueLLMChunkEvent,
|
||||
QueueNodeSucceededEvent,
|
||||
QueueTextChunkEvent,
|
||||
WorkflowQueueMessage,
|
||||
)
|
||||
from core.base.tts.audio_mime import (
|
||||
DEFAULT_TTS_AUDIO_MIME_TYPE,
|
||||
get_model_audio_mime_type,
|
||||
inspect_audio_stream,
|
||||
)
|
||||
from core.credit_usage import CreditUsageAppType, CreditUsageCreatedBy
|
||||
from core.model_manager import ModelManager
|
||||
from graphon.model_runtime.entities.message_entities import TextPromptMessageContent
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from graphon.model_runtime.errors.invoke import InvokeBadRequestError
|
||||
|
||||
|
||||
class AudioTrunk:
|
||||
def __init__(self, status: str, audio, audio_type: str | None = None, error: Exception | None = None):
|
||||
self.audio = audio
|
||||
self.status = status
|
||||
self.audio_type = audio_type
|
||||
self.error = error
|
||||
|
||||
|
||||
class AppGeneratorTTSPublisher:
|
||||
def __init__(
|
||||
self,
|
||||
tenant_id: str,
|
||||
voice: str,
|
||||
language: str | None = None,
|
||||
app_type: CreditUsageAppType = CreditUsageAppType.UNKNOWN,
|
||||
created_by: CreditUsageCreatedBy = CreditUsageCreatedBy.AUDIO,
|
||||
):
|
||||
self.logger = logging.getLogger(__name__)
|
||||
self.tenant_id = tenant_id
|
||||
self.msg_text = ""
|
||||
self._audio_queue: queue.Queue[AudioTrunk] = queue.Queue()
|
||||
self._msg_queue: queue.Queue[WorkflowQueueMessage | MessageQueueMessage | None] = queue.Queue()
|
||||
self.match = re.compile(r"[。.!?]")
|
||||
self.model_manager = ModelManager.for_tenant(
|
||||
tenant_id=self.tenant_id,
|
||||
user_id="responding_tts",
|
||||
request_metadata={"app_type": app_type, "created_by": created_by},
|
||||
)
|
||||
self.model_instance = self.model_manager.get_default_model_instance(
|
||||
tenant_id=self.tenant_id, model_type=ModelType.TTS
|
||||
)
|
||||
self._declared_audio_mime_type = get_model_audio_mime_type(self.model_instance)
|
||||
self.voices = self.model_instance.get_tts_voices(language=language)
|
||||
values = [voice.get("value") for voice in self.voices]
|
||||
self.voice = voice
|
||||
if not voice or voice not in values:
|
||||
self.voice = self.voices[0].get("value")
|
||||
self.max_sentence = 2
|
||||
self._last_audio_event: AudioTrunk | None = None
|
||||
self._cancelled = threading.Event()
|
||||
threading.Thread(target=self._runtime, daemon=True).start()
|
||||
|
||||
def publish(self, message: WorkflowQueueMessage | MessageQueueMessage | None, /):
|
||||
self._msg_queue.put(message)
|
||||
|
||||
def cancel(self):
|
||||
if self._cancelled.is_set():
|
||||
return
|
||||
self._cancelled.set()
|
||||
self._msg_queue.put(None)
|
||||
|
||||
def _runtime(self):
|
||||
audio_type = self._declared_audio_mime_type or DEFAULT_TTS_AUDIO_MIME_TYPE
|
||||
resolved_audio_type: str | None = None
|
||||
try:
|
||||
while True:
|
||||
message = self._msg_queue.get()
|
||||
if self._cancelled.is_set():
|
||||
return
|
||||
text_content = ""
|
||||
incremental_request = False
|
||||
if message is None:
|
||||
text_content = self.msg_text
|
||||
else:
|
||||
match message.event:
|
||||
case QueueAgentMessageEvent() | QueueLLMChunkEvent():
|
||||
message_content = message.event.chunk.delta.message.content
|
||||
if not message_content:
|
||||
continue
|
||||
match message_content:
|
||||
case str():
|
||||
self.msg_text += message_content
|
||||
case list():
|
||||
for content in message_content:
|
||||
if not isinstance(content, TextPromptMessageContent):
|
||||
continue
|
||||
self.msg_text += content.data
|
||||
case QueueTextChunkEvent():
|
||||
self.msg_text += message.event.text
|
||||
case QueueNodeSucceededEvent():
|
||||
if message.event.outputs is None:
|
||||
continue
|
||||
output = message.event.outputs.get("output", "")
|
||||
if isinstance(output, str):
|
||||
self.msg_text += output
|
||||
sentence_arr, text_tmp = self._extract_sentence(self.msg_text)
|
||||
if self._declared_audio_mime_type == DEFAULT_TTS_AUDIO_MIME_TYPE and len(sentence_arr) >= min(
|
||||
self.max_sentence, 7
|
||||
):
|
||||
self.max_sentence += 1
|
||||
text_content = "".join(sentence_arr)
|
||||
self.msg_text = text_tmp
|
||||
incremental_request = True
|
||||
|
||||
if text_content and not text_content.isspace():
|
||||
invoke_result = self.model_instance.invoke_tts(content_text=text_content.strip(), voice=self.voice)
|
||||
audio_stream, next_audio_type = inspect_audio_stream(invoke_result, self._declared_audio_mime_type)
|
||||
if self._cancelled.is_set():
|
||||
return
|
||||
if incremental_request and next_audio_type != DEFAULT_TTS_AUDIO_MIME_TYPE:
|
||||
raise InvokeBadRequestError(
|
||||
"The TTS model declared MP3 but returned a format that cannot be played incrementally"
|
||||
)
|
||||
if resolved_audio_type and resolved_audio_type != next_audio_type:
|
||||
raise InvokeBadRequestError(
|
||||
"TTS provider changed MIME type between audio responses: "
|
||||
f"{resolved_audio_type} then {next_audio_type}"
|
||||
)
|
||||
resolved_audio_type = audio_type = next_audio_type
|
||||
for audio in audio_stream:
|
||||
# ponytail: reads stop at the next chunk; add transport cancellation when Graphon exposes it.
|
||||
if self._cancelled.is_set():
|
||||
return
|
||||
self._audio_queue.put(
|
||||
AudioTrunk(
|
||||
"responding",
|
||||
audio=base64.b64encode(audio),
|
||||
audio_type=audio_type,
|
||||
)
|
||||
)
|
||||
|
||||
if message is None:
|
||||
break
|
||||
except Exception as e:
|
||||
self.logger.warning("TTS generation failed", exc_info=True)
|
||||
self._audio_queue.put(AudioTrunk("error", b"", audio_type=audio_type, error=e))
|
||||
return
|
||||
|
||||
self._audio_queue.put(AudioTrunk("finish", b"", audio_type=audio_type))
|
||||
|
||||
def check_and_get_audio(self, *, block: bool = False):
|
||||
try:
|
||||
if self._last_audio_event and self._last_audio_event.status in {"finish", "error"}:
|
||||
return self._last_audio_event
|
||||
audio = self._audio_queue.get(block=block)
|
||||
self._last_audio_event = audio
|
||||
return audio
|
||||
except queue.Empty:
|
||||
return None
|
||||
|
||||
def _extract_sentence(self, org_text):
|
||||
tx = self.match.finditer(org_text)
|
||||
start = 0
|
||||
result = []
|
||||
for i in tx:
|
||||
end = i.regs[0][1]
|
||||
result.append(org_text[start:end])
|
||||
start = end
|
||||
return result, org_text[start:]
|
||||
@@ -0,0 +1,223 @@
|
||||
import logging
|
||||
from collections.abc import Generator, Iterable, Iterator
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, override
|
||||
|
||||
from core.plugin.entities.plugin_daemon import TTSAudioChunk
|
||||
from graphon.model_runtime.entities.model_entities import ModelPropertyKey
|
||||
from graphon.model_runtime.errors.invoke import InvokeBadRequestError
|
||||
from libs.stream import close_stream
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.model_manager import ModelInstance
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_TTS_AUDIO_MIME_TYPE = "audio/mpeg"
|
||||
_SIGNATURE_SIZE = 32
|
||||
_AUDIO_MIME_TYPE_ALIASES = {
|
||||
"mp3": "audio/mpeg",
|
||||
"audio/mp3": "audio/mpeg",
|
||||
"audio/mpeg": "audio/mpeg",
|
||||
"wav": "audio/wav",
|
||||
"wave": "audio/wav",
|
||||
"audio/wav": "audio/wav",
|
||||
"audio/wave": "audio/wav",
|
||||
"audio/x-wav": "audio/wav",
|
||||
"ogg": "audio/ogg",
|
||||
"oga": "audio/ogg",
|
||||
"audio/ogg": "audio/ogg",
|
||||
"flac": "audio/flac",
|
||||
"audio/flac": "audio/flac",
|
||||
"aac": "audio/aac",
|
||||
"audio/aac": "audio/aac",
|
||||
"m4a": "audio/mp4",
|
||||
"mp4": "audio/mp4",
|
||||
"audio/mp4": "audio/mp4",
|
||||
"webm": "audio/webm",
|
||||
"audio/webm": "audio/webm",
|
||||
}
|
||||
SUPPORTED_TTS_AUDIO_MIME_TYPES = tuple(sorted(set(_AUDIO_MIME_TYPE_ALIASES.values())))
|
||||
|
||||
|
||||
def normalize_audio_mime_type(audio_type: object | None) -> str | None:
|
||||
"""Convert provider audio-type metadata into a browser MIME type."""
|
||||
if not isinstance(audio_type, str):
|
||||
return None
|
||||
|
||||
mime_type = audio_type.split(";", maxsplit=1)[0].strip().lower()
|
||||
return _AUDIO_MIME_TYPE_ALIASES.get(mime_type)
|
||||
|
||||
|
||||
def get_model_audio_mime_type(model_instance: "ModelInstance") -> str | None:
|
||||
"""Read the model's declared TTS MIME type without making it mandatory."""
|
||||
try:
|
||||
model_schema = model_instance.get_model_schema()
|
||||
audio_type = model_schema.model_properties.get(ModelPropertyKey.AUDIO_TYPE)
|
||||
except Exception:
|
||||
logger.debug("Unable to resolve the declared audio type for the TTS model", exc_info=True)
|
||||
return None
|
||||
|
||||
return normalize_audio_mime_type(audio_type)
|
||||
|
||||
|
||||
def sniff_audio_mime_type(audio: bytes | bytearray | memoryview) -> str | None:
|
||||
"""Identify common audio containers from their leading bytes."""
|
||||
signature = bytes(audio[:_SIGNATURE_SIZE])
|
||||
if len(signature) >= 12 and signature[:4] == b"RIFF" and signature[8:12] == b"WAVE":
|
||||
return "audio/wav"
|
||||
if signature.startswith(b"OggS"):
|
||||
return "audio/ogg"
|
||||
if signature.startswith(b"fLaC"):
|
||||
return "audio/flac"
|
||||
if signature.startswith(b"\x1aE\xdf\xa3"):
|
||||
return "audio/webm"
|
||||
if len(signature) >= 8 and signature[4:8] == b"ftyp":
|
||||
return "audio/mp4"
|
||||
if len(signature) >= 2 and signature[0] == 0xFF:
|
||||
if signature[1] & 0xF6 == 0xF0:
|
||||
return "audio/aac"
|
||||
if signature[1] & 0xE0 == 0xE0:
|
||||
return "audio/mpeg"
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_reported_mime_type(mime_type: object | None, source: str) -> str | None:
|
||||
if mime_type is None:
|
||||
return None
|
||||
|
||||
normalized_mime_type = normalize_audio_mime_type(mime_type)
|
||||
if normalized_mime_type is None:
|
||||
raise InvokeBadRequestError(f"TTS provider returned an unsupported {source} MIME type: {mime_type!r}")
|
||||
return normalized_mime_type
|
||||
|
||||
|
||||
def _extract_audio_chunk(chunk: bytes | bytearray | memoryview | TTSAudioChunk) -> tuple[bytes, str | None]:
|
||||
"""Accept old byte chunks and the compatibility carrier for current Graphon."""
|
||||
if isinstance(chunk, TTSAudioChunk):
|
||||
return bytes(chunk), chunk.mime_type
|
||||
|
||||
if isinstance(chunk, (bytes, bytearray, memoryview)):
|
||||
return bytes(chunk), None
|
||||
|
||||
raise InvokeBadRequestError("TTS provider returned a chunk that is not audio bytes")
|
||||
|
||||
|
||||
def resolve_audio_mime_type(
|
||||
audio: bytes | bytearray | memoryview,
|
||||
declared_mime_type: str | None = None,
|
||||
reported_mime_type: str | None = None,
|
||||
) -> str:
|
||||
"""Validate the provider MIME against magic bytes and choose the true type.
|
||||
|
||||
``reported_mime_type`` is emitted with a TTS chunk by current plugin
|
||||
runtimes, and therefore takes precedence over schema metadata. Schema
|
||||
metadata remains a fallback for old plugins that omit the new field.
|
||||
"""
|
||||
normalized_declared_mime_type = _normalize_reported_mime_type(declared_mime_type, "schema")
|
||||
normalized_reported_mime_type = _normalize_reported_mime_type(reported_mime_type, "chunk")
|
||||
detected_mime_type = sniff_audio_mime_type(audio)
|
||||
|
||||
expected_mime_type = normalized_reported_mime_type or normalized_declared_mime_type
|
||||
if expected_mime_type and detected_mime_type and expected_mime_type != detected_mime_type:
|
||||
raise InvokeBadRequestError(
|
||||
"TTS provider output MIME does not match its audio bytes: "
|
||||
f"declared {expected_mime_type}, detected {detected_mime_type}"
|
||||
)
|
||||
|
||||
if normalized_reported_mime_type:
|
||||
if normalized_declared_mime_type and normalized_declared_mime_type != normalized_reported_mime_type:
|
||||
logger.info(
|
||||
"TTS chunk MIME %s overrides schema MIME %s",
|
||||
normalized_reported_mime_type,
|
||||
normalized_declared_mime_type,
|
||||
)
|
||||
return normalized_reported_mime_type
|
||||
|
||||
if detected_mime_type:
|
||||
return detected_mime_type
|
||||
|
||||
return normalized_declared_mime_type or DEFAULT_TTS_AUDIO_MIME_TYPE
|
||||
|
||||
|
||||
type _AudioChunk = bytes | bytearray | memoryview | TTSAudioChunk
|
||||
|
||||
|
||||
class AudioStream(Iterator[bytes]):
|
||||
"""Own a MIME-checked stream, including resources opened while peeking."""
|
||||
|
||||
def __init__(
|
||||
self, chunks: Generator[bytes], *, source: Iterable[_AudioChunk], iterator: Iterator[_AudioChunk]
|
||||
) -> None:
|
||||
self._chunks: Generator[bytes] = chunks
|
||||
self._source: Iterable[_AudioChunk] = source
|
||||
self._iterator: Iterator[_AudioChunk] = iterator
|
||||
self._closed: bool = False
|
||||
|
||||
@override
|
||||
def __next__(self) -> bytes:
|
||||
try:
|
||||
return next(self._chunks)
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
# The validation generator may never have started, but peeking has
|
||||
# already opened the provider. Close its source explicitly as well.
|
||||
close_stream(self._chunks)
|
||||
close_stream(self._iterator)
|
||||
if self._source is not self._iterator:
|
||||
close_stream(self._source)
|
||||
|
||||
|
||||
def inspect_audio_stream(
|
||||
audio_stream: Iterable[_AudioChunk], declared_mime_type: str | None = None
|
||||
) -> tuple[AudioStream, str]:
|
||||
"""Peek and validate without dropping bytes; the returned stream owns cleanup."""
|
||||
iterator: Iterator[_AudioChunk] | None = None
|
||||
leading_chunks: list[bytes] = []
|
||||
signature = bytearray()
|
||||
reported_mime_type: str | None = None
|
||||
|
||||
try:
|
||||
iterator = iter(audio_stream)
|
||||
while len(signature) < _SIGNATURE_SIZE:
|
||||
try:
|
||||
chunk, chunk_mime_type = _extract_audio_chunk(next(iterator))
|
||||
except StopIteration:
|
||||
break
|
||||
normalized_chunk_mime_type = _normalize_reported_mime_type(chunk_mime_type, "chunk")
|
||||
if normalized_chunk_mime_type:
|
||||
if reported_mime_type and reported_mime_type != normalized_chunk_mime_type:
|
||||
raise InvokeBadRequestError(
|
||||
"TTS provider changed MIME type within one audio response: "
|
||||
f"{reported_mime_type} then {normalized_chunk_mime_type}"
|
||||
)
|
||||
reported_mime_type = normalized_chunk_mime_type
|
||||
leading_chunks.append(chunk)
|
||||
signature.extend(chunk[: _SIGNATURE_SIZE - len(signature)])
|
||||
|
||||
mime_type = resolve_audio_mime_type(signature, declared_mime_type, reported_mime_type)
|
||||
except BaseException:
|
||||
close_stream(iterator)
|
||||
if audio_stream is not iterator:
|
||||
close_stream(audio_stream)
|
||||
raise
|
||||
|
||||
def validated_stream() -> Generator[bytes]:
|
||||
for chunk in chain(leading_chunks, iterator):
|
||||
audio, chunk_mime_type = _extract_audio_chunk(chunk)
|
||||
normalized_chunk_mime_type = _normalize_reported_mime_type(chunk_mime_type, "chunk")
|
||||
if normalized_chunk_mime_type and normalized_chunk_mime_type != mime_type:
|
||||
raise InvokeBadRequestError(
|
||||
"TTS provider changed MIME type within one audio response: "
|
||||
f"{mime_type} then {normalized_chunk_mime_type}"
|
||||
)
|
||||
yield audio
|
||||
|
||||
return AudioStream(validated_stream(), source=audio_stream, iterator=iterator), mime_type
|
||||
Reference in new issue
Block a user