chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

+1
View File
@@ -0,0 +1 @@
# Core base package
@@ -0,0 +1,6 @@
from core.base.tts.app_generator_tts_publisher import AppGeneratorTTSPublisher, AudioTrunk
__all__ = [
"AppGeneratorTTSPublisher",
"AudioTrunk",
]
@@ -0,0 +1,175 @@
import base64
import logging
import queue
import re
import threading
from core.app.entities.queue_entities import (
MessageQueueMessage,
QueueAgentMessageEvent,
QueueLLMChunkEvent,
QueueNodeSucceededEvent,
QueueTextChunkEvent,
WorkflowQueueMessage,
)
from core.base.tts.audio_mime import (
DEFAULT_TTS_AUDIO_MIME_TYPE,
get_model_audio_mime_type,
inspect_audio_stream,
)
from core.credit_usage import CreditUsageAppType, CreditUsageCreatedBy
from core.model_manager import ModelManager
from graphon.model_runtime.entities.message_entities import TextPromptMessageContent
from graphon.model_runtime.entities.model_entities import ModelType
from graphon.model_runtime.errors.invoke import InvokeBadRequestError
class AudioTrunk:
def __init__(self, status: str, audio, audio_type: str | None = None, error: Exception | None = None):
self.audio = audio
self.status = status
self.audio_type = audio_type
self.error = error
class AppGeneratorTTSPublisher:
def __init__(
self,
tenant_id: str,
voice: str,
language: str | None = None,
app_type: CreditUsageAppType = CreditUsageAppType.UNKNOWN,
created_by: CreditUsageCreatedBy = CreditUsageCreatedBy.AUDIO,
):
self.logger = logging.getLogger(__name__)
self.tenant_id = tenant_id
self.msg_text = ""
self._audio_queue: queue.Queue[AudioTrunk] = queue.Queue()
self._msg_queue: queue.Queue[WorkflowQueueMessage | MessageQueueMessage | None] = queue.Queue()
self.match = re.compile(r"[。.!?]")
self.model_manager = ModelManager.for_tenant(
tenant_id=self.tenant_id,
user_id="responding_tts",
request_metadata={"app_type": app_type, "created_by": created_by},
)
self.model_instance = self.model_manager.get_default_model_instance(
tenant_id=self.tenant_id, model_type=ModelType.TTS
)
self._declared_audio_mime_type = get_model_audio_mime_type(self.model_instance)
self.voices = self.model_instance.get_tts_voices(language=language)
values = [voice.get("value") for voice in self.voices]
self.voice = voice
if not voice or voice not in values:
self.voice = self.voices[0].get("value")
self.max_sentence = 2
self._last_audio_event: AudioTrunk | None = None
self._cancelled = threading.Event()
threading.Thread(target=self._runtime, daemon=True).start()
def publish(self, message: WorkflowQueueMessage | MessageQueueMessage | None, /):
self._msg_queue.put(message)
def cancel(self):
if self._cancelled.is_set():
return
self._cancelled.set()
self._msg_queue.put(None)
def _runtime(self):
audio_type = self._declared_audio_mime_type or DEFAULT_TTS_AUDIO_MIME_TYPE
resolved_audio_type: str | None = None
try:
while True:
message = self._msg_queue.get()
if self._cancelled.is_set():
return
text_content = ""
incremental_request = False
if message is None:
text_content = self.msg_text
else:
match message.event:
case QueueAgentMessageEvent() | QueueLLMChunkEvent():
message_content = message.event.chunk.delta.message.content
if not message_content:
continue
match message_content:
case str():
self.msg_text += message_content
case list():
for content in message_content:
if not isinstance(content, TextPromptMessageContent):
continue
self.msg_text += content.data
case QueueTextChunkEvent():
self.msg_text += message.event.text
case QueueNodeSucceededEvent():
if message.event.outputs is None:
continue
output = message.event.outputs.get("output", "")
if isinstance(output, str):
self.msg_text += output
sentence_arr, text_tmp = self._extract_sentence(self.msg_text)
if self._declared_audio_mime_type == DEFAULT_TTS_AUDIO_MIME_TYPE and len(sentence_arr) >= min(
self.max_sentence, 7
):
self.max_sentence += 1
text_content = "".join(sentence_arr)
self.msg_text = text_tmp
incremental_request = True
if text_content and not text_content.isspace():
invoke_result = self.model_instance.invoke_tts(content_text=text_content.strip(), voice=self.voice)
audio_stream, next_audio_type = inspect_audio_stream(invoke_result, self._declared_audio_mime_type)
if self._cancelled.is_set():
return
if incremental_request and next_audio_type != DEFAULT_TTS_AUDIO_MIME_TYPE:
raise InvokeBadRequestError(
"The TTS model declared MP3 but returned a format that cannot be played incrementally"
)
if resolved_audio_type and resolved_audio_type != next_audio_type:
raise InvokeBadRequestError(
"TTS provider changed MIME type between audio responses: "
f"{resolved_audio_type} then {next_audio_type}"
)
resolved_audio_type = audio_type = next_audio_type
for audio in audio_stream:
# ponytail: reads stop at the next chunk; add transport cancellation when Graphon exposes it.
if self._cancelled.is_set():
return
self._audio_queue.put(
AudioTrunk(
"responding",
audio=base64.b64encode(audio),
audio_type=audio_type,
)
)
if message is None:
break
except Exception as e:
self.logger.warning("TTS generation failed", exc_info=True)
self._audio_queue.put(AudioTrunk("error", b"", audio_type=audio_type, error=e))
return
self._audio_queue.put(AudioTrunk("finish", b"", audio_type=audio_type))
def check_and_get_audio(self, *, block: bool = False):
try:
if self._last_audio_event and self._last_audio_event.status in {"finish", "error"}:
return self._last_audio_event
audio = self._audio_queue.get(block=block)
self._last_audio_event = audio
return audio
except queue.Empty:
return None
def _extract_sentence(self, org_text):
tx = self.match.finditer(org_text)
start = 0
result = []
for i in tx:
end = i.regs[0][1]
result.append(org_text[start:end])
start = end
return result, org_text[start:]
@@ -0,0 +1,223 @@
import logging
from collections.abc import Generator, Iterable, Iterator
from itertools import chain
from typing import TYPE_CHECKING, override
from core.plugin.entities.plugin_daemon import TTSAudioChunk
from graphon.model_runtime.entities.model_entities import ModelPropertyKey
from graphon.model_runtime.errors.invoke import InvokeBadRequestError
from libs.stream import close_stream
if TYPE_CHECKING:
from core.model_manager import ModelInstance
logger = logging.getLogger(__name__)
DEFAULT_TTS_AUDIO_MIME_TYPE = "audio/mpeg"
_SIGNATURE_SIZE = 32
_AUDIO_MIME_TYPE_ALIASES = {
"mp3": "audio/mpeg",
"audio/mp3": "audio/mpeg",
"audio/mpeg": "audio/mpeg",
"wav": "audio/wav",
"wave": "audio/wav",
"audio/wav": "audio/wav",
"audio/wave": "audio/wav",
"audio/x-wav": "audio/wav",
"ogg": "audio/ogg",
"oga": "audio/ogg",
"audio/ogg": "audio/ogg",
"flac": "audio/flac",
"audio/flac": "audio/flac",
"aac": "audio/aac",
"audio/aac": "audio/aac",
"m4a": "audio/mp4",
"mp4": "audio/mp4",
"audio/mp4": "audio/mp4",
"webm": "audio/webm",
"audio/webm": "audio/webm",
}
SUPPORTED_TTS_AUDIO_MIME_TYPES = tuple(sorted(set(_AUDIO_MIME_TYPE_ALIASES.values())))
def normalize_audio_mime_type(audio_type: object | None) -> str | None:
"""Convert provider audio-type metadata into a browser MIME type."""
if not isinstance(audio_type, str):
return None
mime_type = audio_type.split(";", maxsplit=1)[0].strip().lower()
return _AUDIO_MIME_TYPE_ALIASES.get(mime_type)
def get_model_audio_mime_type(model_instance: "ModelInstance") -> str | None:
"""Read the model's declared TTS MIME type without making it mandatory."""
try:
model_schema = model_instance.get_model_schema()
audio_type = model_schema.model_properties.get(ModelPropertyKey.AUDIO_TYPE)
except Exception:
logger.debug("Unable to resolve the declared audio type for the TTS model", exc_info=True)
return None
return normalize_audio_mime_type(audio_type)
def sniff_audio_mime_type(audio: bytes | bytearray | memoryview) -> str | None:
"""Identify common audio containers from their leading bytes."""
signature = bytes(audio[:_SIGNATURE_SIZE])
if len(signature) >= 12 and signature[:4] == b"RIFF" and signature[8:12] == b"WAVE":
return "audio/wav"
if signature.startswith(b"OggS"):
return "audio/ogg"
if signature.startswith(b"fLaC"):
return "audio/flac"
if signature.startswith(b"\x1aE\xdf\xa3"):
return "audio/webm"
if len(signature) >= 8 and signature[4:8] == b"ftyp":
return "audio/mp4"
if len(signature) >= 2 and signature[0] == 0xFF:
if signature[1] & 0xF6 == 0xF0:
return "audio/aac"
if signature[1] & 0xE0 == 0xE0:
return "audio/mpeg"
return None
def _normalize_reported_mime_type(mime_type: object | None, source: str) -> str | None:
if mime_type is None:
return None
normalized_mime_type = normalize_audio_mime_type(mime_type)
if normalized_mime_type is None:
raise InvokeBadRequestError(f"TTS provider returned an unsupported {source} MIME type: {mime_type!r}")
return normalized_mime_type
def _extract_audio_chunk(chunk: bytes | bytearray | memoryview | TTSAudioChunk) -> tuple[bytes, str | None]:
"""Accept old byte chunks and the compatibility carrier for current Graphon."""
if isinstance(chunk, TTSAudioChunk):
return bytes(chunk), chunk.mime_type
if isinstance(chunk, (bytes, bytearray, memoryview)):
return bytes(chunk), None
raise InvokeBadRequestError("TTS provider returned a chunk that is not audio bytes")
def resolve_audio_mime_type(
audio: bytes | bytearray | memoryview,
declared_mime_type: str | None = None,
reported_mime_type: str | None = None,
) -> str:
"""Validate the provider MIME against magic bytes and choose the true type.
``reported_mime_type`` is emitted with a TTS chunk by current plugin
runtimes, and therefore takes precedence over schema metadata. Schema
metadata remains a fallback for old plugins that omit the new field.
"""
normalized_declared_mime_type = _normalize_reported_mime_type(declared_mime_type, "schema")
normalized_reported_mime_type = _normalize_reported_mime_type(reported_mime_type, "chunk")
detected_mime_type = sniff_audio_mime_type(audio)
expected_mime_type = normalized_reported_mime_type or normalized_declared_mime_type
if expected_mime_type and detected_mime_type and expected_mime_type != detected_mime_type:
raise InvokeBadRequestError(
"TTS provider output MIME does not match its audio bytes: "
f"declared {expected_mime_type}, detected {detected_mime_type}"
)
if normalized_reported_mime_type:
if normalized_declared_mime_type and normalized_declared_mime_type != normalized_reported_mime_type:
logger.info(
"TTS chunk MIME %s overrides schema MIME %s",
normalized_reported_mime_type,
normalized_declared_mime_type,
)
return normalized_reported_mime_type
if detected_mime_type:
return detected_mime_type
return normalized_declared_mime_type or DEFAULT_TTS_AUDIO_MIME_TYPE
type _AudioChunk = bytes | bytearray | memoryview | TTSAudioChunk
class AudioStream(Iterator[bytes]):
"""Own a MIME-checked stream, including resources opened while peeking."""
def __init__(
self, chunks: Generator[bytes], *, source: Iterable[_AudioChunk], iterator: Iterator[_AudioChunk]
) -> None:
self._chunks: Generator[bytes] = chunks
self._source: Iterable[_AudioChunk] = source
self._iterator: Iterator[_AudioChunk] = iterator
self._closed: bool = False
@override
def __next__(self) -> bytes:
try:
return next(self._chunks)
except BaseException:
self.close()
raise
def close(self) -> None:
if self._closed:
return
self._closed = True
# The validation generator may never have started, but peeking has
# already opened the provider. Close its source explicitly as well.
close_stream(self._chunks)
close_stream(self._iterator)
if self._source is not self._iterator:
close_stream(self._source)
def inspect_audio_stream(
audio_stream: Iterable[_AudioChunk], declared_mime_type: str | None = None
) -> tuple[AudioStream, str]:
"""Peek and validate without dropping bytes; the returned stream owns cleanup."""
iterator: Iterator[_AudioChunk] | None = None
leading_chunks: list[bytes] = []
signature = bytearray()
reported_mime_type: str | None = None
try:
iterator = iter(audio_stream)
while len(signature) < _SIGNATURE_SIZE:
try:
chunk, chunk_mime_type = _extract_audio_chunk(next(iterator))
except StopIteration:
break
normalized_chunk_mime_type = _normalize_reported_mime_type(chunk_mime_type, "chunk")
if normalized_chunk_mime_type:
if reported_mime_type and reported_mime_type != normalized_chunk_mime_type:
raise InvokeBadRequestError(
"TTS provider changed MIME type within one audio response: "
f"{reported_mime_type} then {normalized_chunk_mime_type}"
)
reported_mime_type = normalized_chunk_mime_type
leading_chunks.append(chunk)
signature.extend(chunk[: _SIGNATURE_SIZE - len(signature)])
mime_type = resolve_audio_mime_type(signature, declared_mime_type, reported_mime_type)
except BaseException:
close_stream(iterator)
if audio_stream is not iterator:
close_stream(audio_stream)
raise
def validated_stream() -> Generator[bytes]:
for chunk in chain(leading_chunks, iterator):
audio, chunk_mime_type = _extract_audio_chunk(chunk)
normalized_chunk_mime_type = _normalize_reported_mime_type(chunk_mime_type, "chunk")
if normalized_chunk_mime_type and normalized_chunk_mime_type != mime_type:
raise InvokeBadRequestError(
"TTS provider changed MIME type within one audio response: "
f"{mime_type} then {normalized_chunk_mime_type}"
)
yield audio
return AudioStream(validated_stream(), source=audio_stream, iterator=iterator), mime_type