chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1,204 @@
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
import type { ReactElement } from 'react'
import { screen, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { consoleQuery } from '@/service/console'
import { renderWithConsoleQuery } from '@/test/console/query-data'
import EmbeddingProcess from '../index'
const status: DocumentStatusResponse = {
id: 'document-1',
indexing_status: 'completed',
completed_segments: 1,
total_segments: 1,
processing_started_at: null,
parsing_completed_at: null,
cleaning_completed_at: null,
splitting_completed_at: null,
completed_at: null,
paused_at: null,
error: null,
stopped_at: null,
}
let deploymentEdition: 'CLOUD' | 'COMMUNITY' = 'COMMUNITY'
let mockPlanType: 'sandbox' | 'professional' | 'team' = 'sandbox'
let mockPollingState: {
statusList: DocumentStatusResponse[]
isEmbedding: boolean
isEmbeddingCompleted: boolean
} = {
statusList: [],
isEmbedding: false,
isEmbeddingCompleted: false,
}
vi.mock('@/hooks/use-api-access-url', () => ({
useDatasetApiAccessUrl: () => 'https://api.example.com/docs',
}))
vi.mock('../use-indexing-status-polling', () => ({
useIndexingStatusPolling: () => mockPollingState,
}))
vi.mock('../upgrade-banner', () => ({
default: () => <div>upgrade processing priority</div>,
}))
function render(ui: ReactElement) {
return renderWithConsoleQuery(ui, {
systemFeatures: { deployment_edition: deploymentEdition },
features: { billing: { subscription: { plan: mockPlanType } } },
})
}
describe('EmbeddingProcess', () => {
beforeEach(() => {
vi.clearAllMocks()
deploymentEdition = 'COMMUNITY'
mockPlanType = 'sandbox'
mockPollingState = {
statusList: [],
isEmbedding: false,
isEmbeddingCompleted: false,
}
})
it('shows that document indexing is in progress', () => {
mockPollingState.isEmbedding = true
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
expect(screen.getByRole('status')).toHaveTextContent('datasetDocuments.embedding.processing')
})
it('shows that document indexing has completed', () => {
mockPollingState.isEmbeddingCompleted = true
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
expect(screen.getByText('datasetDocuments.embedding.completed')).toHaveAttribute(
'role',
'status',
)
})
it('shows the vector-space admission alert after processing completes', () => {
mockPollingState = {
statusList: [
{
...status,
id: 'document-1',
indexing_status: 'error',
error_code: 'vector_space_estimate_exceeded',
estimated_vector_space_mb: 61,
vector_space_limit_mb: 50,
},
],
isEmbedding: false,
isEmbeddingCompleted: true,
}
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
expect(screen.getByText('datasetDocuments.embedding.completed')).toHaveAttribute(
'role',
'status',
)
const alert = screen.getByRole('alert')
expect(alert).toHaveTextContent(
'datasetDocuments.embedding.vectorSpaceEstimateExceeded.description',
)
expect(alert).toHaveTextContent('61')
expect(alert).toHaveTextContent('50')
expect(within(alert).queryByRole('button')).not.toBeInTheDocument()
})
it('does not show the vector-space alert for another indexing error', () => {
mockPollingState = {
statusList: [
{
...status,
id: 'document-1',
indexing_status: 'error',
error_code: null,
estimated_vector_space_mb: 61,
vector_space_limit_mb: 50,
},
],
isEmbedding: false,
isEmbeddingCompleted: true,
}
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
expect(screen.queryByRole('alert')).not.toBeInTheDocument()
})
it('does not suggest an upgrade to team users', () => {
deploymentEdition = 'CLOUD'
mockPlanType = 'team'
mockPollingState = {
statusList: [
{
...status,
id: 'document-1',
indexing_status: 'error',
error_code: 'vector_space_estimate_exceeded',
estimated_vector_space_mb: 61,
vector_space_limit_mb: 50,
},
],
isEmbedding: false,
isEmbeddingCompleted: true,
}
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
const alert = screen.getByRole('alert')
expect(alert).toHaveTextContent(
'datasetDocuments.embedding.vectorSpaceEstimateExceeded.description',
)
expect(alert).toHaveTextContent('61')
expect(alert).toHaveTextContent('50')
expect(within(alert).queryByRole('button')).not.toBeInTheDocument()
})
it('links to the document list and invalidates its cache on activation', async () => {
const user = userEvent.setup()
const { queryClient } = render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
const documentListKey = consoleQuery.datasets.byDatasetId.documents.get.queryKey({
input: { params: { dataset_id: 'dataset-1' } },
})
queryClient.setQueryData(documentListKey, {
data: [],
has_more: false,
page: 1,
limit: 20,
total: 0,
})
const link = screen.getByRole('link', { name: 'datasetCreation.stepThree.navTo' })
expect(link).toHaveAttribute('href', '/datasets/dataset-1/documents')
await user.click(link)
expect(queryClient.getQueryState(documentListKey)?.isInvalidated).toBe(true)
})
it('links to the dataset API reference', () => {
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
expect(screen.getByRole('link', { name: 'Access the API' })).toHaveAttribute(
'href',
'https://api.example.com/docs',
)
})
it('offers a processing-priority upgrade outside the team plan', () => {
deploymentEdition = 'CLOUD'
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
expect(screen.getByText('upgrade processing priority')).toBeInTheDocument()
})
})
@@ -0,0 +1,95 @@
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
import { render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { DataSourceType } from '@/models/datasets'
import IndexingProgressItem from '../indexing-progress-item'
// Billing labels have their own tests and are outside indexing progress semantics.
vi.mock('@/app/components/billing/priority-label', () => ({
default: () => <span data-testid="priority-label">Priority</span>,
}))
describe('IndexingProgressItem', () => {
beforeEach(() => {
vi.clearAllMocks()
})
const makeDetail = (overrides: Partial<DocumentStatusResponse> = {}): DocumentStatusResponse => ({
id: 'doc-1',
indexing_status: 'indexing',
processing_started_at: 0,
parsing_completed_at: 0,
cleaning_completed_at: 0,
splitting_completed_at: 0,
completed_at: null,
paused_at: null,
error: null,
stopped_at: null,
completed_segments: 50,
total_segments: 100,
...overrides,
})
it('should render name and progress for embedding status', () => {
render(
<IndexingProgressItem
detail={makeDetail()}
name="test.pdf"
sourceType={DataSourceType.FILE}
/>,
)
expect(screen.getByText('test.pdf')).toBeInTheDocument()
expect(screen.getByText('50%')).toBeInTheDocument()
expect(screen.getByRole('progressbar', { name: 'test.pdf' })).toHaveAttribute(
'aria-valuenow',
'50',
)
})
it('announces a file completing after processing without retaining its progress bar', () => {
const { rerender } = render(<IndexingProgressItem detail={makeDetail()} name="done.pdf" />)
const status = screen.getByRole('status')
expect(status).toBeEmptyDOMElement()
expect(screen.getByRole('progressbar', { name: 'done.pdf' })).toBeInTheDocument()
rerender(
<IndexingProgressItem
detail={makeDetail({ indexing_status: 'completed' })}
name="done.pdf"
/>,
)
expect(status).toHaveTextContent('done.pdf: datasetDocuments.embedding.completed')
expect(screen.queryByRole('progressbar')).not.toBeInTheDocument()
})
it('opens the full indexing error from the warning button', async () => {
const user = userEvent.setup()
render(
<IndexingProgressItem
detail={makeDetail({ indexing_status: 'error', error: 'Parse failed' })}
name="broken.pdf"
/>,
)
expect(screen.getByRole('status')).toHaveTextContent('broken.pdf: common.error: Parse failed')
await user.click(screen.getByRole('button', { name: 'common.error' }))
expect(await screen.findByRole('dialog')).toHaveTextContent('Parse failed')
await user.keyboard('{Escape}')
expect(screen.getByRole('button', { name: 'common.error' })).toHaveFocus()
})
it('should use the localized fallback when an error has no message', async () => {
const user = userEvent.setup()
render(
<IndexingProgressItem
detail={makeDetail({ indexing_status: 'error', error: null })}
name="broken.pdf"
/>,
)
await user.click(screen.getByRole('button', { name: 'common.error' }))
expect(await screen.findByRole('dialog')).toHaveTextContent('common.error')
})
})
@@ -0,0 +1,71 @@
import type { ProcessRuleResponse } from '@dify/contracts/api/console/datasets/types.gen'
import { render, screen } from '@testing-library/react'
import { ProcessMode } from '@/models/datasets'
import { RETRIEVE_METHOD } from '@/types/app'
import RuleDetail from '../rule-detail'
const rule: ProcessRuleResponse = {
limits: { indexing_max_segmentation_tokens_length: 4000 },
mode: ProcessMode.general,
rules: {
segmentation: { separator: '\n', max_tokens: 500, chunk_overlap: 50 },
pre_processing_rules: [
{ id: 'remove_extra_spaces', enabled: true },
{ id: 'remove_urls_emails', enabled: false },
],
},
}
it('summarizes the custom segmentation and only enabled cleaning rules', () => {
render(
<RuleDetail
sourceData={rule}
indexingType="high_quality"
retrievalMethod={RETRIEVE_METHOD.semantic}
/>,
)
expect(screen.getByText('datasetDocuments.embedding.custom')).toBeInTheDocument()
expect(screen.getByText('500')).toBeInTheDocument()
expect(screen.getByText('datasetCreation.stepTwo.removeExtraSpaces')).toBeInTheDocument()
expect(screen.queryByText('datasetCreation.stepTwo.removeUrlEmails')).not.toBeInTheDocument()
expect(screen.getByText('datasetCreation.stepTwo.qualified')).toBeInTheDocument()
expect(screen.getByText('dataset.retrieval.semantic_search.title')).toBeInTheDocument()
})
it('shows parent and child chunk limits for hierarchical processing', () => {
render(
<RuleDetail
sourceData={{
...rule,
mode: ProcessMode.parentChild,
rules: {
parent_mode: 'paragraph',
segmentation: { separator: '\n', max_tokens: 1000 },
subchunk_segmentation: { separator: '\n', max_tokens: 200 },
},
}}
indexingType="high_quality"
/>,
)
expect(
screen.getByText('datasetDocuments.embedding.hierarchical · dataset.parentMode.paragraph'),
).toBeInTheDocument()
expect(
screen.getByText(
'datasetDocuments.embedding.parentMaxTokens 1000; datasetDocuments.embedding.childMaxTokens 200',
),
).toBeInTheDocument()
})
it('uses keyword retrieval for economical indexing', () => {
render(
<RuleDetail
sourceData={rule}
indexingType="economy"
retrievalMethod={RETRIEVE_METHOD.semantic}
/>,
)
expect(screen.getByText('datasetCreation.stepTwo.economical')).toBeInTheDocument()
expect(screen.getByText('dataset.retrieval.keyword_search.title')).toBeInTheDocument()
expect(screen.queryByText('dataset.retrieval.semantic_search.title')).not.toBeInTheDocument()
})
@@ -0,0 +1,170 @@
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
import { act, renderHook } from '@testing-library/react'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { useIndexingStatusPolling } from '../use-indexing-status-polling'
const mockFetchIndexingStatusBatch = vi.fn()
vi.mock('@/service/datasets', () => ({
fetchIndexingStatusBatch: (...args: unknown[]) => mockFetchIndexingStatusBatch(...args),
}))
const status = (
indexing_status: DocumentStatusResponse['indexing_status'],
overrides: Partial<DocumentStatusResponse> = {},
): DocumentStatusResponse => ({
id: 'doc-1',
indexing_status,
processing_started_at: null,
parsing_completed_at: null,
cleaning_completed_at: null,
splitting_completed_at: null,
completed_at: null,
paused_at: null,
error: null,
stopped_at: null,
completed_segments: 0,
total_segments: 0,
...overrides,
})
describe('useIndexingStatusPolling', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.clearAllMocks()
})
afterEach(() => {
vi.useRealTimers()
})
const defaultParams = { datasetId: 'ds-1', batchId: 'batch-1' }
it('should fetch status on mount and update state', async () => {
mockFetchIndexingStatusBatch.mockResolvedValue({
data: [status('indexing', { completed_segments: 5, total_segments: 10 })],
})
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
// Flush the resolved promise
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledWith({
datasetId: 'ds-1',
batchId: 'batch-1',
})
expect(result.current.statusList).toHaveLength(1)
expect(result.current.isEmbedding).toBe(true)
expect(result.current.isEmbeddingCompleted).toBe(false)
await act(async () => {
await vi.advanceTimersByTimeAsync(2499)
})
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(1)
await act(async () => {
await vi.advanceTimersByTimeAsync(1)
})
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(2)
})
it('should stop polling when all completed', async () => {
mockFetchIndexingStatusBatch.mockResolvedValue({
data: [status('completed')],
})
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
expect(result.current.isEmbeddingCompleted).toBe(true)
expect(result.current.isEmbedding).toBe(false)
// Should not schedule another poll
const callCount = mockFetchIndexingStatusBatch.mock.calls.length
await act(async () => {
await vi.advanceTimersByTimeAsync(5000)
})
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(callCount)
})
it('should continue polling on fetch error', async () => {
mockFetchIndexingStatusBatch.mockRejectedValueOnce(new Error('network')).mockResolvedValueOnce({
data: [status('completed')],
})
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
// First call: rejects
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
// Advance past polling interval for retry
await act(async () => {
await vi.advanceTimersByTimeAsync(2500)
})
expect(result.current.isEmbeddingCompleted).toBe(true)
})
it('should detect embedding statuses', async () => {
mockFetchIndexingStatusBatch.mockResolvedValue({
data: [status('splitting'), status('parsing')],
})
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
expect(result.current.isEmbedding).toBe(true)
expect(result.current.isEmbeddingCompleted).toBe(false)
})
it('should detect mixed statuses (some completed, some embedding)', async () => {
mockFetchIndexingStatusBatch.mockResolvedValue({
data: [status('completed'), status('indexing')],
})
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
expect(result.current.statusList).toHaveLength(2)
expect(result.current.isEmbedding).toBe(true)
expect(result.current.isEmbeddingCompleted).toBe(false)
})
it('should cleanup on unmount', async () => {
mockFetchIndexingStatusBatch.mockResolvedValue({
data: [status('indexing')],
})
const { unmount } = renderHook(() => useIndexingStatusPolling(defaultParams))
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
const callCount = mockFetchIndexingStatusBatch.mock.calls.length
unmount()
await act(async () => {
await vi.advanceTimersByTimeAsync(5000)
})
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(callCount)
})
it('should treat error and paused as completed statuses', async () => {
mockFetchIndexingStatusBatch.mockResolvedValue({
data: [status('error'), status('paused')],
})
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
expect(result.current.isEmbeddingCompleted).toBe(true)
expect(result.current.isEmbedding).toBe(false)
})
})
@@ -0,0 +1,134 @@
import type {
DocumentResponse,
DocumentStatusResponse,
} from '@dify/contracts/api/console/datasets/types.gen'
import { describe, expect, it } from 'vite-plus/test'
import { createDocumentLookup, getFileType, getSourcePercent, isSourceEmbedding } from '../utils'
const status = (overrides: Partial<DocumentStatusResponse> = {}): DocumentStatusResponse => ({
id: 'doc-1',
indexing_status: 'indexing',
processing_started_at: null,
parsing_completed_at: null,
cleaning_completed_at: null,
splitting_completed_at: null,
completed_at: null,
paused_at: null,
error: null,
stopped_at: null,
completed_segments: 0,
total_segments: 0,
...overrides,
})
describe('isSourceEmbedding', () => {
const embeddingStatuses = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const
const nonEmbeddingStatuses = ['completed', 'error', 'paused'] as const
it.each(embeddingStatuses)('should return true for status "%s"', (indexing_status) => {
expect(isSourceEmbedding(status({ indexing_status }))).toBe(true)
})
it.each(nonEmbeddingStatuses)('should return false for status "%s"', (indexing_status) => {
expect(isSourceEmbedding(status({ indexing_status }))).toBe(false)
})
})
describe('getSourcePercent', () => {
it('should calculate correct percentage', () => {
expect(getSourcePercent(status({ completed_segments: 50, total_segments: 100 }))).toBe(50)
})
it('should return 0 when total is 0', () => {
expect(getSourcePercent(status({ completed_segments: 0, total_segments: 0 }))).toBe(0)
})
it('should cap at 100', () => {
expect(getSourcePercent(status({ completed_segments: 150, total_segments: 100 }))).toBe(100)
})
it('should round to nearest integer', () => {
expect(getSourcePercent(status({ completed_segments: 1, total_segments: 3 }))).toBe(33)
})
it('treats absent segment totals as zero', () => {
expect(getSourcePercent(status({ completed_segments: null, total_segments: null }))).toBe(0)
})
})
describe('getFileType', () => {
it('should extract extension from filename', () => {
expect(getFileType('document.pdf')).toBe('pdf')
})
it('should return last extension for multi-dot names', () => {
expect(getFileType('archive.tar.gz')).toBe('gz')
})
it('should default to "txt" for undefined', () => {
expect(getFileType(undefined)).toBe('txt')
})
it('should default to "txt" for empty string', () => {
expect(getFileType('')).toBe('txt')
})
})
describe('createDocumentLookup', () => {
const documents: DocumentResponse[] = [
{
data_source_detail_dict: {},
id: 'doc-1',
name: 'test.pdf',
data_source_type: 'upload_file',
data_source_info: {
upload_file: { id: 'f1', name: 'test.pdf' },
notion_page_icon: undefined,
},
},
{
data_source_detail_dict: {},
id: 'doc-2',
name: 'notion-page',
data_source_type: 'notion_import',
data_source_info: {
upload_file: { id: 'f2', name: '' },
notion_page_icon: 'https://icon.url',
},
},
]
it('should get name by id', () => {
const lookup = createDocumentLookup(documents)
expect(lookup.getName('doc-1')).toBe('test.pdf')
})
it('should get source type by id', () => {
const lookup = createDocumentLookup(documents)
expect(lookup.getSourceType('doc-1')).toBe('upload_file')
})
it('should get notion icon for legacy data source', () => {
const lookup = createDocumentLookup(documents)
expect(lookup.getNotionIcon('doc-2')).toBe('https://icon.url')
})
it('should return undefined notion icon for non-legacy info', () => {
const docs: DocumentResponse[] = [
{
id: 'doc-3',
name: 'New source',
data_source_detail_dict: {},
data_source_info: { some_other: 'field' },
},
]
const lookup = createDocumentLookup(docs)
expect(lookup.getNotionIcon('doc-3')).toBeUndefined()
})
it('should handle empty documents list', () => {
const lookup = createDocumentLookup([])
expect(lookup.getDocument('any')).toBeUndefined()
expect(lookup.getName('any')).toBeUndefined()
})
})
@@ -0,0 +1,181 @@
import type { DocumentResponse } from '@dify/contracts/api/console/datasets/types.gen'
import type { FC } from 'react'
import type { RETRIEVE_METHOD } from '@/types/app'
import { buttonVariants } from '@langgenius/dify-ui/button'
import { cn } from '@langgenius/dify-ui/cn'
import { Separator } from '@langgenius/dify-ui/separator'
import { RiArrowRightLine, RiLoader2Fill, RiTerminalBoxLine } from '@remixicon/react'
import { useQuery, useQueryClient } from '@tanstack/react-query'
import { useAtomValue } from 'jotai'
import { useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import VectorSpaceAdmissionAlert from '@/app/components/datasets/common/vector-space-admission-alert'
import { deploymentEditionAtom } from '@/features/system-features/state'
import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url'
import Link from '@/next/link'
import { consoleQuery } from '@/service/console'
import IndexingProgressItem from './indexing-progress-item'
import RuleDetail from './rule-detail'
import UpgradeBanner from './upgrade-banner'
import { useIndexingStatusPolling } from './use-indexing-status-polling'
import { createDocumentLookup } from './utils'
type EmbeddingProcessProps = {
datasetId: string
batchId: string
documents?: DocumentResponse[]
indexingType?: string
retrievalMethod?: RETRIEVE_METHOD
}
// Status header component
const StatusHeader: FC<{ isEmbedding: boolean; isCompleted: boolean }> = ({
isEmbedding,
isCompleted,
}) => {
const { t } = useTranslation(['datasetDocuments'])
return (
<div
role="status"
aria-atomic="true"
className="flex items-center gap-x-1 system-md-semibold-uppercase text-text-secondary"
>
{isEmbedding && (
<>
<RiLoader2Fill aria-hidden="true" className="size-4 animate-spin" />
<span>{t(($) => $['embedding.processing'], { ns: 'datasetDocuments' })}</span>
</>
)}
{isCompleted && t(($) => $['embedding.completed'], { ns: 'datasetDocuments' })}
</div>
)
}
// Action buttons component
const ActionButtons: FC<{
apiReferenceUrl: string
documentsHref: string
onNavigateToDocuments: () => void
}> = ({ apiReferenceUrl, documentsHref, onNavigateToDocuments }) => {
const { t } = useTranslation(['datasetCreation'])
return (
<div className="mt-6 flex flex-wrap items-center gap-2 py-2">
<Link
href={apiReferenceUrl}
target="_blank"
rel="noopener noreferrer"
className={cn(buttonVariants(), 'w-fit')}
>
<RiTerminalBoxLine className="size-4" />
<span>Access the API</span>
</Link>
<Link
href={documentsHref}
className={cn(buttonVariants({ variant: 'primary' }), 'w-fit')}
onClick={onNavigateToDocuments}
>
<span>{t(($) => $['stepThree.navTo'], { ns: 'datasetCreation' })}</span>
<RiArrowRightLine className="size-4 stroke-current stroke-1" />
</Link>
</div>
)
}
const EmbeddingProcess: FC<EmbeddingProcessProps> = ({
datasetId,
batchId,
documents = [],
indexingType,
retrievalMethod,
}) => {
const deploymentEdition = useAtomValue(deploymentEditionAtom)
const { data: plan } = useQuery(
consoleQuery.features.get.queryOptions({
enabled: deploymentEdition === 'CLOUD',
select: (data) => data.billing.subscription.plan,
}),
)
const queryClient = useQueryClient()
const invalidDocumentList = () =>
queryClient.invalidateQueries({
queryKey: consoleQuery.datasets.byDatasetId.documents.get.key(),
})
const apiReferenceUrl = useDatasetApiAccessUrl()
// Polling hook for indexing status
const { statusList, isEmbedding, isEmbeddingCompleted } = useIndexingStatusPolling({
datasetId,
batchId,
})
// Get process rule for the first document
const firstDocumentId = documents[0]?.id
const { data: ruleDetail } = useQuery(
consoleQuery.datasets.processRule.get.queryOptions({
input: { query: { document_id: firstDocumentId } },
enabled: !!firstDocumentId,
}),
)
// Document lookup utilities - memoized for performance
const documentLookup = useMemo(() => createDocumentLookup(documents), [documents])
const documentsHref = `/datasets/${datasetId}/documents`
const showUpgradeBanner =
deploymentEdition === 'CLOUD' && (plan === 'sandbox' || plan === 'professional')
const showVectorSpaceUpgrade =
deploymentEdition === 'CLOUD' && (plan === 'sandbox' || plan === 'professional')
const vectorSpaceAdmissionError = statusList.find(
(detail) => detail.error_code === 'vector_space_estimate_exceeded',
)
return (
<>
<div className="flex flex-col gap-y-3">
<StatusHeader isEmbedding={isEmbedding} isCompleted={isEmbeddingCompleted} />
{vectorSpaceAdmissionError?.estimated_vector_space_mb != null &&
vectorSpaceAdmissionError.vector_space_limit_mb != null && (
<VectorSpaceAdmissionAlert
showUpgrade={showVectorSpaceUpgrade}
estimatedMb={vectorSpaceAdmissionError.estimated_vector_space_mb}
planLimitMb={vectorSpaceAdmissionError.vector_space_limit_mb}
/>
)}
{showUpgradeBanner && <UpgradeBanner />}
<div className="flex flex-col gap-0.5 pb-2">
{statusList.map((detail) => (
<IndexingProgressItem
key={detail.id}
detail={detail}
name={documentLookup.getName(detail.id)}
sourceType={documentLookup.getSourceType(detail.id)}
notionIcon={documentLookup.getNotionIcon(detail.id)}
/>
))}
</div>
<Separator orientation="horizontal" className="my-0 h-[0.5px] bg-divider-subtle" />
<RuleDetail
sourceData={ruleDetail}
indexingType={indexingType}
retrievalMethod={retrievalMethod}
/>
</div>
<ActionButtons
apiReferenceUrl={apiReferenceUrl}
documentsHref={documentsHref}
onNavigateToDocuments={invalidDocumentList}
/>
</>
)
}
export default EmbeddingProcess
@@ -0,0 +1,128 @@
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
import type { ComponentProps, FC } from 'react'
import { cn } from '@langgenius/dify-ui/cn'
import { Infotip, InfotipContent, InfotipTrigger } from '@langgenius/dify-ui/infotip'
import { RiCheckboxCircleFill } from '@remixicon/react'
import { useId } from 'react'
import { useTranslation } from 'react-i18next'
import NotionIcon from '@/app/components/base/notion-icon'
import PriorityLabel from '@/app/components/billing/priority-label'
import { DataSourceType } from '@/models/datasets'
import DocumentFileIcon from '../../common/document-file-icon'
import { getFileType, getSourcePercent, isSourceEmbedding } from './utils'
type IndexingProgressItemProps = {
detail: DocumentStatusResponse
name?: string
sourceType?: DataSourceType
notionIcon?: ComponentProps<typeof NotionIcon>['src']
}
// Status icon component for completed/error states
const StatusIcon: FC<{ status: string; error?: string }> = ({ status, error }) => {
const { t } = useTranslation(['common'])
if (status === 'completed')
return <RiCheckboxCircleFill aria-hidden="true" className="size-4 shrink-0 text-text-success" />
if (status === 'error') {
const errorLabel = error || t(($) => $.error, { ns: 'common' })
return (
<Infotip>
<InfotipTrigger
aria-label={t(($) => $.error, { ns: 'common' })}
iconVariant="warning"
iconSize="large"
className="text-text-destructive"
/>
<InfotipContent
aria-label={t(($) => $.error, { ns: 'common' })}
className="whitespace-pre-wrap"
sideOffset={4}
>
{errorLabel}
</InfotipContent>
</Infotip>
)
}
return null
}
// Source type icon component
const SourceTypeIcon: FC<{
sourceType?: DataSourceType
name?: string
notionIcon?: ComponentProps<typeof NotionIcon>['src']
}> = ({ sourceType, name, notionIcon }) => {
if (sourceType === DataSourceType.FILE) {
return (
<DocumentFileIcon size="sm" className="shrink-0" name={name} extension={getFileType(name)} />
)
}
if (sourceType === DataSourceType.NOTION) {
return <NotionIcon className="shrink-0" type="page" src={notionIcon} />
}
return null
}
const IndexingProgressItem: FC<IndexingProgressItemProps> = ({
detail,
name,
sourceType,
notionIcon,
}) => {
const { t } = useTranslation(['common', 'datasetDocuments'])
const nameId = useId()
const isEmbedding = isSourceEmbedding(detail)
const percent = getSourcePercent(detail)
const isError = detail.indexing_status === 'error'
const statusText =
detail.indexing_status === 'completed'
? t(($) => $['embedding.completed'], { ns: 'datasetDocuments' })
: isError
? `${t(($) => $.error, { ns: 'common' })}: ${detail.error || ''}`
: detail.indexing_status === 'paused'
? t(($) => $['embedding.paused'], { ns: 'datasetDocuments' })
: ''
return (
<div
className={cn(
'relative h-6.5 overflow-hidden rounded-md bg-components-progress-bar-bg',
isError && 'bg-state-destructive-hover-alt',
)}
>
{isEmbedding && (
<div
className="absolute top-0 left-0 h-full min-w-0.5 border-r-2 border-r-components-progress-bar-progress-highlight bg-components-progress-bar-progress"
role="progressbar"
aria-labelledby={nameId}
aria-valuemin={0}
aria-valuemax={100}
aria-valuenow={percent}
style={{ width: `${percent}%` }}
/>
)}
<span role="status" aria-atomic="true" className="sr-only">
{statusText && `${name}: ${statusText}`}
</span>
<div className="z-1 flex h-full items-center gap-1 pr-2 pl-1.5">
<SourceTypeIcon sourceType={sourceType} name={name} notionIcon={notionIcon} />
<div className="flex w-0 grow items-center gap-1" title={name}>
<div id={nameId} className="truncate system-xs-medium text-text-secondary">
{name}
</div>
<PriorityLabel className="ml-0" />
</div>
{isEmbedding && <div className="shrink-0 text-xs text-text-secondary">{`${percent}%`}</div>}
<StatusIcon status={detail.indexing_status} error={detail.error ?? undefined} />
</div>
</div>
)
}
export default IndexingProgressItem
@@ -0,0 +1,146 @@
import type { ProcessRuleResponse } from '@dify/contracts/api/console/datasets/types.gen'
import type { FC } from 'react'
import { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
import { ProcessMode } from '@/models/datasets'
import { RETRIEVE_METHOD } from '@/types/app'
import { indexMethodIcon, retrievalIcon } from '../icons'
import { IndexingType } from '../step-two'
type RuleDetailProps = {
sourceData?: ProcessRuleResponse
indexingType?: string
retrievalMethod?: RETRIEVE_METHOD
}
// Lookup table for pre-processing rule names
const PRE_PROCESSING_RULE_KEYS = {
remove_extra_spaces: 'stepTwo.removeExtraSpaces',
remove_urls_emails: 'stepTwo.removeUrlEmails',
remove_stopwords: 'stepTwo.removeStopwords',
} as const
// Lookup table for retrieval method icons
const RETRIEVAL_ICON_MAP: Partial<Record<RETRIEVE_METHOD, string>> = {
[RETRIEVE_METHOD.fullText]: retrievalIcon.fullText,
[RETRIEVE_METHOD.hybrid]: retrievalIcon.hybrid,
[RETRIEVE_METHOD.semantic]: retrievalIcon.vector,
[RETRIEVE_METHOD.keywordSearch]: retrievalIcon.fullText,
}
const isNumber = (value: unknown): value is number => typeof value === 'number'
const RuleDetail: FC<RuleDetailProps> = ({ sourceData, indexingType, retrievalMethod }) => {
const { t } = useTranslation([
'dataset',
'datasetCreation',
'datasetDocuments',
'datasetSettings',
])
const segmentationRuleLabels = {
mode: t(($) => $['embedding.mode'], { ns: 'datasetDocuments' }),
segmentLength: t(($) => $['embedding.segmentLength'], { ns: 'datasetDocuments' }),
textCleaning: t(($) => $['embedding.textCleaning'], { ns: 'datasetDocuments' }),
}
const getRuleName = useCallback(
(key: string): string | undefined => {
const translationKey = PRE_PROCESSING_RULE_KEYS[key as keyof typeof PRE_PROCESSING_RULE_KEYS]
return translationKey ? t(($) => $[translationKey], { ns: 'datasetCreation' }) : undefined
},
[t],
)
const getModeValue = useCallback((): string => {
if (!sourceData?.mode) return '-'
if (sourceData.mode === ProcessMode.general)
return t(($) => $['embedding.custom'], { ns: 'datasetDocuments' })
const parentModeLabel =
sourceData.rules?.parent_mode === 'paragraph'
? t(($) => $['parentMode.paragraph'], { ns: 'dataset' })
: t(($) => $['parentMode.fullDoc'], { ns: 'dataset' })
return `${t(($) => $['embedding.hierarchical'], { ns: 'datasetDocuments' })} · ${parentModeLabel}`
}, [sourceData, t])
const getSegmentLengthValue = useCallback((): string | number => {
if (!sourceData?.mode) return '-'
const maxTokens = isNumber(sourceData.rules?.segmentation?.max_tokens)
? sourceData.rules.segmentation.max_tokens
: '-'
if (sourceData.mode === ProcessMode.general) return maxTokens
const childMaxTokens = isNumber(sourceData.rules?.subchunk_segmentation?.max_tokens)
? sourceData.rules.subchunk_segmentation.max_tokens
: '-'
return `${t(($) => $['embedding.parentMaxTokens'], { ns: 'datasetDocuments' })} ${maxTokens}; ${t(($) => $['embedding.childMaxTokens'], { ns: 'datasetDocuments' })} ${childMaxTokens}`
}, [sourceData, t])
const getTextCleaningValue = useCallback((): string => {
if (!sourceData?.mode) return '-'
const enabledRules =
sourceData.rules?.pre_processing_rules?.filter((rule) => rule.enabled) || []
const ruleNames = enabledRules
.map((rule) => {
const name = getRuleName(rule.id)
return typeof name === 'string' ? name : ''
})
.filter((name) => name)
return ruleNames.length > 0 ? ruleNames.join(',') : '-'
}, [sourceData, getRuleName])
const fieldValueGetters: Record<string, () => string | number> = {
mode: getModeValue,
segmentLength: getSegmentLengthValue,
textCleaning: getTextCleaningValue,
}
const isEconomical = indexingType === IndexingType.ECONOMICAL
const indexMethodIconSrc = isEconomical
? indexMethodIcon.economical
: indexMethodIcon.high_quality
const indexModeLabel = t(($) => $[`stepTwo.${isEconomical ? 'economical' : 'qualified'}`], {
ns: 'datasetCreation',
})
const effectiveRetrievalMethod = isEconomical
? 'keyword_search'
: (retrievalMethod ?? 'semantic_search')
const retrievalLabel = t(($) => $[`retrieval.${effectiveRetrievalMethod}.title`], {
ns: 'dataset',
})
const retrievalIconSrc =
RETRIEVAL_ICON_MAP[retrievalMethod as keyof typeof RETRIEVAL_ICON_MAP] ?? retrievalIcon.vector
return (
<div className="flex flex-col gap-1">
{Object.keys(segmentationRuleLabels).map((field) => (
<FieldInfo
key={field}
label={segmentationRuleLabels[field as keyof typeof segmentationRuleLabels]}
displayedValue={String(fieldValueGetters[field]!())}
/>
))}
<FieldInfo
label={t(($) => $['stepTwo.indexMode'], { ns: 'datasetCreation' })}
displayedValue={indexModeLabel}
valueIcon={<img className="size-4" src={indexMethodIconSrc} alt="" />}
/>
<FieldInfo
label={t(($) => $['form.retrievalSetting.title'], { ns: 'datasetSettings' })}
displayedValue={retrievalLabel}
valueIcon={<img className="size-4" src={retrievalIconSrc} alt="" />}
/>
</div>
)
}
export default RuleDetail
@@ -0,0 +1,24 @@
import type { FC } from 'react'
import { useTranslation } from 'react-i18next'
import UpgradeBtn from '@/app/components/billing/upgrade-btn'
const UpgradeBanner: FC = () => {
const { t } = useTranslation(['billing'])
return (
<div className="flex h-14 items-center rounded-xl border-[0.5px] border-black/5 bg-white p-3 shadow-md">
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-[#FFF6ED]">
<span
aria-hidden
className="i-custom-vender-solid-general-zap-fast h-4 w-4 text-[#FB6514]"
/>
</div>
<div className="mx-3 grow text-[13px] font-medium text-gray-700">
{t(($) => $['plansCommon.documentProcessingPriorityUpgrade'], { ns: 'billing' })}
</div>
<UpgradeBtn loc="knowledge-speed-up" />
</div>
)
}
export default UpgradeBanner
@@ -0,0 +1,87 @@
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
import { useEffect, useRef, useState } from 'react'
import { fetchIndexingStatusBatch } from '@/service/datasets'
const POLLING_INTERVAL = 2500
const COMPLETED_STATUSES = ['completed', 'error', 'paused'] as const
const EMBEDDING_STATUSES = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const
type IndexingStatusPollingParams = {
datasetId: string
batchId: string
}
type IndexingStatusPollingResult = {
statusList: DocumentStatusResponse[]
isEmbedding: boolean
isEmbeddingCompleted: boolean
}
const isStatusCompleted = (status: string): boolean =>
COMPLETED_STATUSES.includes(status as (typeof COMPLETED_STATUSES)[number])
const isAllCompleted = (statusList: DocumentStatusResponse[]): boolean =>
statusList.every((item) => isStatusCompleted(item.indexing_status))
/**
* Custom hook for polling indexing status with automatic stop on completion.
* Handles the polling lifecycle and provides derived states for UI rendering.
*/
export const useIndexingStatusPolling = ({
datasetId,
batchId,
}: IndexingStatusPollingParams): IndexingStatusPollingResult => {
const [statusList, setStatusList] = useState<DocumentStatusResponse[]>([])
const isStopPollingRef = useRef(false)
useEffect(() => {
// Reset polling state on mount
isStopPollingRef.current = false
let timeoutId: ReturnType<typeof setTimeout> | null = null
const fetchStatus = async (): Promise<DocumentStatusResponse[]> => {
const response = await fetchIndexingStatusBatch({ datasetId, batchId })
setStatusList(response.data)
return response.data
}
const poll = async (): Promise<void> => {
if (isStopPollingRef.current) return
try {
const data = await fetchStatus()
if (isAllCompleted(data)) {
isStopPollingRef.current = true
return
}
} catch {
// Continue polling on error
}
if (!isStopPollingRef.current) {
timeoutId = setTimeout(() => {
poll()
}, POLLING_INTERVAL)
}
}
poll()
return () => {
isStopPollingRef.current = true
if (timeoutId) clearTimeout(timeoutId)
}
}, [datasetId, batchId])
const isEmbedding = statusList.some((item) =>
EMBEDDING_STATUSES.includes(item?.indexing_status as (typeof EMBEDDING_STATUSES)[number]),
)
const isEmbeddingCompleted = statusList.length > 0 && isAllCompleted(statusList)
return {
statusList,
isEmbedding,
isEmbeddingCompleted,
}
}
@@ -0,0 +1,49 @@
import type {
DocumentResponse,
DocumentStatusResponse,
} from '@dify/contracts/api/console/datasets/types.gen'
import { toCreationDataSourceType } from '@/app/components/datasets/dataset-fields'
import { getNotionPageIcon } from '@/app/components/datasets/documents/document-source'
const EMBEDDING_STATUSES = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const
/**
* Check if a status indicates the source is being embedded
*/
export const isSourceEmbedding = (detail: DocumentStatusResponse): boolean =>
EMBEDDING_STATUSES.includes(detail.indexing_status as (typeof EMBEDDING_STATUSES)[number])
/**
* Calculate the progress percentage for a document
*/
export const getSourcePercent = (detail: DocumentStatusResponse): number => {
const completedCount = detail.completed_segments || 0
const totalCount = detail.total_segments || 0
if (totalCount === 0) return 0
const percent = Math.round((completedCount * 100) / totalCount)
return Math.min(percent, 100)
}
/**
* Get file extension from filename, defaults to 'txt'
*/
export const getFileType = (name?: string): string => name?.split('.').pop() || 'txt'
/**
* Document lookup utilities - provides document info by ID from a list
*/
export const createDocumentLookup = (documents: DocumentResponse[]) => {
const documentMap = new Map(documents.map((doc) => [doc.id, doc]))
return {
getDocument: (id: string) => documentMap.get(id),
getName: (id: string) => documentMap.get(id)?.name,
getSourceType: (id: string) => toCreationDataSourceType(documentMap.get(id)?.data_source_type),
getNotionIcon: (id: string) => getNotionPageIcon(documentMap.get(id)?.data_source_info, false),
}
}