chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
@@ -0,0 +1,112 @@
|
||||
"""Celery entry points of the trace pipeline, routed by ``OPS_TRACE_QUEUE``.
|
||||
|
||||
Delivery is at most once per sink: an event is redelivered only while its
|
||||
records are not persisted yet, or when the Agent collector could not decide
|
||||
ownership (enterprise telemetry is then skipped, it already ran).
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from celery import shared_task
|
||||
from dify_trace_sdk.events import TraceEvent
|
||||
|
||||
from extensions.ext_application_services import application_services
|
||||
from extensions.ext_storage import storage
|
||||
from services.tracing.adapters import PAYLOAD_PREFIX
|
||||
from services.tracing.delivery import ExportJob
|
||||
from services.tracing.errors import SourceNotFoundError, SourceNotReadyError, TraceCollectorUnavailableError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REDELIVERY_LIMIT = 10
|
||||
PAYLOAD_RETENTION = timedelta(days=2)
|
||||
_PAYLOAD_DAY = re.compile(rf"{PAYLOAD_PREFIX}/(\d{{4}}-\d{{2}}-\d{{2}})/")
|
||||
|
||||
|
||||
def _expired(key: str, cutoff: date) -> bool:
|
||||
match = _PAYLOAD_DAY.match(key)
|
||||
if match is None:
|
||||
return False
|
||||
try:
|
||||
return date.fromisoformat(match.group(1)) < cutoff
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _redelivery_delay(retries: int) -> int:
|
||||
return min(60, 5 * 2**retries)
|
||||
|
||||
|
||||
def _delete_quietly(key: str) -> None:
|
||||
try:
|
||||
storage.delete(key)
|
||||
except Exception:
|
||||
logger.warning("Trace payload cleanup failed: key=%s", key, exc_info=True)
|
||||
|
||||
|
||||
@shared_task(bind=True, max_retries=REDELIVERY_LIMIT)
|
||||
def deliver_trace_event(self, envelope: dict[str, Any]) -> None:
|
||||
payload_key = envelope.get("payload_key")
|
||||
redelivering = False
|
||||
try:
|
||||
raw = storage.load(payload_key) if isinstance(payload_key, str) else envelope["event"]
|
||||
event = TraceEvent.model_validate_json(raw)
|
||||
except Exception:
|
||||
logger.warning("Trace event unreadable; dropped", exc_info=True)
|
||||
if isinstance(payload_key, str):
|
||||
_delete_quietly(payload_key)
|
||||
return
|
||||
try:
|
||||
application_services().tracing.delivery.deliver(
|
||||
event, enterprise_delivered=envelope.get("enterprise_delivered") is True
|
||||
)
|
||||
except SourceNotFoundError:
|
||||
logger.info("Trace event source not found; dropped: kind=%s app_id=%s", event.payload.kind, event.app_id)
|
||||
except SourceNotReadyError:
|
||||
if self.request.retries < REDELIVERY_LIMIT:
|
||||
redelivering = True
|
||||
raise self.retry(args=(envelope,), countdown=_redelivery_delay(self.request.retries)) from None
|
||||
logger.warning("Trace event source never became ready; dropped: kind=%s", event.payload.kind)
|
||||
except TraceCollectorUnavailableError:
|
||||
if self.request.retries < REDELIVERY_LIMIT:
|
||||
redelivering = True
|
||||
raise self.retry(
|
||||
args=({**envelope, "enterprise_delivered": True},),
|
||||
countdown=_redelivery_delay(self.request.retries),
|
||||
) from None
|
||||
logger.warning("Agent trace collector unavailable; dropped: kind=%s", event.payload.kind)
|
||||
finally:
|
||||
if isinstance(payload_key, str) and not redelivering:
|
||||
_delete_quietly(payload_key)
|
||||
|
||||
|
||||
@shared_task
|
||||
def retry_trace_export(job_key: str) -> None:
|
||||
try:
|
||||
job = ExportJob.model_validate_json(storage.load(job_key))
|
||||
except Exception:
|
||||
logger.warning("Trace export job unreadable; dropped: key=%s", job_key, exc_info=True)
|
||||
_delete_quietly(job_key)
|
||||
return
|
||||
try:
|
||||
application_services().tracing.delivery.retry(job)
|
||||
finally:
|
||||
_delete_quietly(job_key)
|
||||
|
||||
|
||||
@shared_task
|
||||
def sweep_trace_payloads() -> None:
|
||||
"""Delete expired new-pipeline payloads; legacy workers still own their payloads during an upgrade."""
|
||||
cutoff = (datetime.now(UTC) - PAYLOAD_RETENTION).date()
|
||||
# Directory listings need the trailing slash on some storage backends.
|
||||
prefix = f"{PAYLOAD_PREFIX}/"
|
||||
try:
|
||||
keys = storage.scan(prefix, files=True)
|
||||
except (FileNotFoundError, NotImplementedError):
|
||||
return
|
||||
for key in keys:
|
||||
if key.startswith(prefix) and _expired(key, cutoff):
|
||||
_delete_quietly(key)
|
||||
Reference in new issue
Block a user