chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1,662 @@
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
import type { CloudPlan } from '@dify/contracts/api/console/features/types.gen'
import type { PipelineDocument } from '@dify/contracts/api/console/rag/types.gen'
import type { ReactElement } from 'react'
import type { Mock } from 'vite-plus/test'
import { QueryClient } from '@tanstack/react-query'
import { act, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { NuqsTestingAdapter } from 'nuqs/adapters/testing'
import * as React from 'react'
import { IndexingType } from '@/app/components/datasets/create/step-two'
import { DatasourceType } from '@/models/pipeline'
import { consoleQuery } from '@/service/console'
import { createConsoleQueryWrapper } from '@/test/console/query-data'
import { render as renderWithConsoleState } from '@/test/console/render'
import { RETRIEVE_METHOD } from '@/types/app'
import EmbeddingProcess from '../index'
// Mock provider context
let deploymentEdition: 'CLOUD' | 'COMMUNITY' = 'COMMUNITY'
let mockPlanType: CloudPlan = 'sandbox'
vi.mock('@/app/components/datasets/common/vector-space-admission-alert', () => ({
default: ({
showUpgrade,
estimatedMb,
planLimitMb,
}: {
showUpgrade: boolean
estimatedMb: number
planLimitMb: number
}) => (
<div>{`vector space admission alert ${estimatedMb}MB / ${planLimitMb}MB ${
showUpgrade ? 'with upgrade' : 'without upgrade'
}`}</div>
),
}))
// Mock the batch indexing status request
let mockFetchIndexingStatus: Mock
let mockIndexingStatusData: DocumentStatusResponse[] = []
// Mock useDatasetApiAccessUrl hook
vi.mock('@/hooks/use-api-access-url', () => ({
useDatasetApiAccessUrl: () => 'https://docs.dify.ai/api-reference/datasets',
}))
// Test Data Factory Functions
/**
* Creates a mock InitialDocumentDetail for testing
* Uses deterministic counter-based IDs to avoid flaky tests
*/
let documentIdCounter = 0
const createMockDocument = (overrides: Partial<PipelineDocument> = {}): PipelineDocument => ({
id: overrides.id ?? `doc-${++documentIdCounter}`,
name: 'test-document.txt',
data_source_type: DatasourceType.localFile,
data_source_info: {},
enabled: true,
error: '',
indexing_status: 'waiting',
position: 0,
...overrides,
})
/**
* Creates a mock DocumentStatusResponse for testing
*/
const createMockIndexingStatus = (
overrides: Partial<DocumentStatusResponse> = {},
): DocumentStatusResponse => ({
id: `status-${++documentIdCounter}`,
indexing_status: 'waiting',
processing_started_at: 1700000000,
parsing_completed_at: 0,
cleaning_completed_at: 0,
splitting_completed_at: 0,
completed_at: null,
paused_at: null,
error: null,
stopped_at: null,
completed_segments: 0,
total_segments: 100,
...overrides,
})
/**
* Creates default props for EmbeddingProcess component
*/
const createDefaultProps = (
overrides: Partial<{
datasetId: string
batchId: string
documents: PipelineDocument[]
indexingType: IndexingType
retrievalMethod: RETRIEVE_METHOD
}> = {},
) => ({
datasetId: 'dataset-123',
batchId: 'batch-456',
documents: [createMockDocument({ id: 'doc-1', name: 'test-doc.pdf' })],
indexingType: IndexingType.QUALIFIED,
retrievalMethod: RETRIEVE_METHOD.semantic,
...overrides,
})
function render(ui: ReactElement) {
const { wrapper: QueryWrapper } = createConsoleQueryWrapper({
systemFeatures: { deployment_edition: deploymentEdition },
features: { billing: { subscription: { plan: mockPlanType } } },
})
return renderWithConsoleState(ui, {
wrapper: ({ children }) => (
<NuqsTestingAdapter>
<QueryWrapper>{children}</QueryWrapper>
</NuqsTestingAdapter>
),
})
}
describe('EmbeddingProcess', () => {
beforeEach(() => {
vi.spyOn(QueryClient.prototype, 'invalidateQueries')
vi.clearAllMocks()
vi.useFakeTimers({ shouldAdvanceTime: true })
// Reset deterministic ID counter for reproducible tests
documentIdCounter = 0
// Reset mock states
deploymentEdition = 'COMMUNITY'
mockPlanType = 'sandbox'
mockIndexingStatusData = []
// Setup default mock for fetchIndexingStatus
mockFetchIndexingStatus = vi
.fn()
.mockImplementation(() => Promise.resolve({ data: mockIndexingStatusData }))
})
afterEach(() => {
vi.useRealTimers()
})
describe('Rendering', () => {
it('should render API reference link with correct URL', () => {
const props = createDefaultProps()
render(<EmbeddingProcess {...props} />)
const apiLink = screen.getByRole('link', { name: /access the api/i })
expect(apiLink).toHaveAttribute('href', 'https://docs.dify.ai/api-reference/datasets')
expect(apiLink).toHaveAttribute('target', '_blank')
expect(apiLink).toHaveAttribute('rel', 'noopener noreferrer')
})
})
// Billing/Upgrade Banner Tests
describe('Billing and Upgrade Banner', () => {
// Tests for billing-related UI
it('should not show upgrade banner when billing is disabled', () => {
deploymentEdition = 'COMMUNITY'
const props = createDefaultProps()
render(<EmbeddingProcess {...props} />)
expect(
screen.queryByText('billing.plansCommon.documentProcessingPriorityUpgrade'),
).not.toBeInTheDocument()
})
it('should show upgrade banner when billing is enabled and plan is not team', () => {
deploymentEdition = 'CLOUD'
mockPlanType = 'sandbox'
const props = createDefaultProps()
render(<EmbeddingProcess {...props} />)
expect(
screen.getByText('billing.plansCommon.documentProcessingPriorityUpgrade'),
).toBeInTheDocument()
})
it('should not show upgrade banner when plan is team', () => {
deploymentEdition = 'CLOUD'
mockPlanType = 'team'
const props = createDefaultProps()
render(<EmbeddingProcess {...props} />)
expect(
screen.queryByText('billing.plansCommon.documentProcessingPriorityUpgrade'),
).not.toBeInTheDocument()
})
it('should show upgrade banner for professional plan', () => {
deploymentEdition = 'CLOUD'
mockPlanType = 'professional'
const props = createDefaultProps()
render(<EmbeddingProcess {...props} />)
expect(
screen.getByText('billing.plansCommon.documentProcessingPriorityUpgrade'),
).toBeInTheDocument()
})
})
// Status Display Tests
describe('Status Display', () => {
// Tests for embedding status display
it('should show waiting status when all documents are waiting', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'waiting' }),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.waiting')).toBeInTheDocument()
})
it('should show processing status when any document is indexing', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'indexing' }),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.processing')).toBeInTheDocument()
})
it('should show processing status when any document is splitting', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'splitting' }),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.processing')).toBeInTheDocument()
})
it('should show processing status when any document is parsing', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'parsing' }),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.processing')).toBeInTheDocument()
})
it('should show processing status when any document is cleaning', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'cleaning' }),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.processing')).toBeInTheDocument()
})
it('should show completed status when all documents are completed', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'completed' }),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.completed')).toBeInTheDocument()
})
it('should show the vector-space admission alert after processing completes', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({
id: 'doc-1',
indexing_status: 'error',
error: 'Processing failed',
error_code: 'vector_space_estimate_exceeded',
estimated_vector_space_mb: 61,
vector_space_limit_mb: 50,
}),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.completed')).toBeInTheDocument()
expect(
screen.getByText('vector space admission alert 61MB / 50MB without upgrade'),
).toBeInTheDocument()
})
it('should not show the vector-space alert for another indexing error', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({
id: 'doc-1',
indexing_status: 'error',
error_code: null,
estimated_vector_space_mb: 61,
vector_space_limit_mb: 50,
}),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.queryByText(/vector space admission alert/)).not.toBeInTheDocument()
})
it('should not suggest an upgrade to team users', async () => {
deploymentEdition = 'CLOUD'
mockPlanType = 'team'
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({
id: 'doc-1',
indexing_status: 'error',
error_code: 'vector_space_estimate_exceeded',
estimated_vector_space_mb: 61,
vector_space_limit_mb: 50,
}),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(
screen.getByText('vector space admission alert 61MB / 50MB without upgrade'),
).toBeInTheDocument()
})
it('should show completed status when all documents are paused', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'paused' }),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.completed')).toBeInTheDocument()
})
})
// Progress Bar Tests
describe('Progress Display', () => {
// Tests for progress bar rendering
it('should show progress percentage for embedding documents', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({
id: 'doc-1',
indexing_status: 'indexing',
completed_segments: 50,
total_segments: 100,
}),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('50%')).toBeInTheDocument()
})
it('should cap progress at 100%', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({
id: 'doc-1',
indexing_status: 'indexing',
completed_segments: 150,
total_segments: 100,
}),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('100%')).toBeInTheDocument()
})
it('should show 0% when total_segments is 0', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({
id: 'doc-1',
indexing_status: 'indexing',
completed_segments: 0,
total_segments: 0,
}),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('0%')).toBeInTheDocument()
})
it('should not show progress for completed documents', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({
id: 'doc-1',
indexing_status: 'completed',
completed_segments: 100,
total_segments: 100,
}),
]
const props = createDefaultProps({ documents: [doc1] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.queryByText('100%')).not.toBeInTheDocument()
})
})
// Polling Logic Tests
describe('Polling Logic', () => {
// Tests for API polling behavior
it('polls until processing completes, then stops scheduling requests', async () => {
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'indexing' }),
]
render(<EmbeddingProcess {...createDefaultProps()} />)
await screen.findByText('datasetDocuments.embedding.processing')
expect(mockFetchIndexingStatus).toHaveBeenCalledWith({
params: { dataset_id: 'dataset-123', batch: 'batch-456' },
})
const initialCount = mockFetchIndexingStatus.mock.calls.length
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'completed' }),
]
await act(async () => {
await vi.advanceTimersByTimeAsync(2500)
})
expect(mockFetchIndexingStatus.mock.calls.length).toBeGreaterThan(initialCount)
expect(screen.getByText('datasetDocuments.embedding.completed')).toBeInTheDocument()
const completedCount = mockFetchIndexingStatus.mock.calls.length
await act(async () => {
await vi.advanceTimersByTimeAsync(7500)
})
expect(mockFetchIndexingStatus).toHaveBeenCalledTimes(completedCount)
})
it.each(['error', 'paused'] as const)('stops polling when documents are %s', async (status) => {
mockIndexingStatusData = [createMockIndexingStatus({ id: 'doc-1', indexing_status: status })]
render(<EmbeddingProcess {...createDefaultProps()} />)
await screen.findByText('datasetDocuments.embedding.completed')
await act(async () => {
await vi.advanceTimersByTimeAsync(1)
})
const terminalCount = mockFetchIndexingStatus.mock.calls.length
await act(async () => {
await vi.advanceTimersByTimeAsync(7500)
})
expect(mockFetchIndexingStatus).toHaveBeenCalledTimes(terminalCount)
})
it('should cleanup timeout on unmount', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'indexing' }),
]
const props = createDefaultProps({ documents: [doc1] })
const { unmount } = render(<EmbeddingProcess {...props} />)
// Wait for initial fetch
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
const callCountBeforeUnmount = mockFetchIndexingStatus.mock.calls.length
// Unmount before next poll
unmount()
// Advance timer
await act(async () => {
await vi.advanceTimersByTimeAsync(5000)
})
// Assert - should not poll after unmount
expect(mockFetchIndexingStatus.mock.calls.length).toBe(callCountBeforeUnmount)
})
})
describe('User Interactions', () => {
it('should link to the document list and invalidate its cache on activation', async () => {
const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime })
const props = createDefaultProps({ datasetId: 'my-dataset-123' })
render(<EmbeddingProcess {...props} />)
const link = screen.getByRole('link', { name: 'datasetCreation.stepThree.navTo' })
expect(link).toHaveAttribute('href', '/datasets/my-dataset-123/documents')
await user.click(link)
expect(QueryClient.prototype.invalidateQueries).toHaveBeenCalledWith({
queryKey: consoleQuery.datasets.byDatasetId.documents.get.key(),
})
})
})
// Document Display Tests
describe('Document Display', () => {
// Tests for document list rendering
it('should display multiple documents', async () => {
const doc1 = createMockDocument({ id: 'doc-1', name: 'file1.txt' })
const doc2 = createMockDocument({ id: 'doc-2', name: 'file2.pdf' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'indexing' }),
createMockIndexingStatus({ id: 'doc-2', indexing_status: 'waiting' }),
]
const props = createDefaultProps({ documents: [doc1, doc2] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('file1.txt')).toBeInTheDocument()
expect(screen.getByText('file2.pdf')).toBeInTheDocument()
})
})
describe('Edge Cases', () => {
// Tests for boundary conditions
it('should handle empty indexing status response', async () => {
mockIndexingStatusData = []
const props = createDefaultProps()
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
// Assert - should not show any status text when empty
expect(screen.queryByText('datasetDocuments.embedding.waiting')).not.toBeInTheDocument()
expect(screen.queryByText('datasetDocuments.embedding.processing')).not.toBeInTheDocument()
expect(screen.queryByText('datasetDocuments.embedding.completed')).not.toBeInTheDocument()
})
it('should handle mixed status documents', async () => {
const doc1 = createMockDocument({ id: 'doc-1' })
const doc2 = createMockDocument({ id: 'doc-2' })
const doc3 = createMockDocument({ id: 'doc-3' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'completed' }),
createMockIndexingStatus({ id: 'doc-2', indexing_status: 'indexing' }),
createMockIndexingStatus({ id: 'doc-3', indexing_status: 'error' }),
]
const props = createDefaultProps({ documents: [doc1, doc2, doc3] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
// Assert - should show processing (since one is still indexing)
expect(screen.getByText('datasetDocuments.embedding.processing')).toBeInTheDocument()
})
})
describe('Mixed terminal statuses', () => {
// Tests for useMemo computed values
it('shows completion when all documents are completed, failed, or paused', async () => {
// Arrange - completed + error + paused = all terminal
const doc1 = createMockDocument({ id: 'doc-1' })
const doc2 = createMockDocument({ id: 'doc-2' })
const doc3 = createMockDocument({ id: 'doc-3' })
mockIndexingStatusData = [
createMockIndexingStatus({ id: 'doc-1', indexing_status: 'completed' }),
createMockIndexingStatus({ id: 'doc-2', indexing_status: 'error' }),
createMockIndexingStatus({ id: 'doc-3', indexing_status: 'paused' }),
]
const props = createDefaultProps({ documents: [doc1, doc2, doc3] })
render(<EmbeddingProcess {...props} />)
await waitFor(() => {
expect(mockFetchIndexingStatus).toHaveBeenCalled()
})
expect(screen.getByText('datasetDocuments.embedding.completed')).toBeInTheDocument()
})
})
})
vi.mock('@/service/console', async (importOriginal) => {
const actual = await importOriginal<typeof import('@/service/console')>()
const { withConsoleResponses } = await import('@/test/console/service-responses')
return {
...actual,
consoleQuery: withConsoleResponses(actual.consoleQuery, {
'datasets.processRule.get': async () => ({
mode: 'custom',
rules: { parent_mode: 'paragraph' },
}),
'datasets.byDatasetId.batch.byBatch.indexingStatus.get': (input) =>
mockFetchIndexingStatus(input),
}),
}
})
@@ -0,0 +1,302 @@
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
import type { PipelineDocument } from '@dify/contracts/api/console/rag/types.gen'
import type { IndexingType } from '@/app/components/datasets/create/step-two'
import type { DataSourceNotionPage } from '@/models/common'
import type { RETRIEVE_METHOD } from '@/types/app'
import { buttonVariants } from '@langgenius/dify-ui/button'
import { cn } from '@langgenius/dify-ui/cn'
import { Popover, PopoverContent, PopoverTrigger } from '@langgenius/dify-ui/popover'
import { Separator } from '@langgenius/dify-ui/separator'
import {
RiAedFill,
RiArrowRightLine,
RiCheckboxCircleFill,
RiErrorWarningFill,
RiLoader2Fill,
RiTerminalBoxLine,
} from '@remixicon/react'
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
import { useAtomValue } from 'jotai'
import * as React from 'react'
import { useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import NotionIcon from '@/app/components/base/notion-icon'
import PriorityLabel from '@/app/components/billing/priority-label'
import UpgradeBtn from '@/app/components/billing/upgrade-btn'
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
import VectorSpaceAdmissionAlert from '@/app/components/datasets/common/vector-space-admission-alert'
import { deploymentEditionAtom } from '@/features/system-features/state'
import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url'
import { DatasourceType } from '@/models/pipeline'
import Link from '@/next/link'
import { consoleQuery } from '@/service/console'
import RuleDetail from './rule-detail'
const isNotionPageIcon = (value: unknown): value is DataSourceNotionPage['page_icon'] =>
typeof value === 'object' && value !== null && 'type' in value
type EmbeddingProcessProps = {
datasetId: string
batchId: string
documents?: PipelineDocument[]
indexingType?: IndexingType
retrievalMethod?: RETRIEVE_METHOD
}
const EmbeddingProcess = ({
datasetId,
batchId,
documents = [],
indexingType,
retrievalMethod,
}: EmbeddingProcessProps) => {
const { t } = useTranslation(['billing', 'datasetCreation', 'datasetDocuments'])
const deploymentEdition = useAtomValue(deploymentEditionAtom)
const { data: plan } = useQuery(
consoleQuery.features.get.queryOptions({
enabled: deploymentEdition === 'CLOUD',
select: (data) => data.billing.subscription.plan,
}),
)
const [indexingStatusBatchDetail, setIndexingStatusDetail] = useState<DocumentStatusResponse[]>(
[],
)
const [shouldPoll, setShouldPoll] = useState(true)
const { mutateAsync: fetchIndexingStatus } = useMutation(
consoleQuery.datasets.byDatasetId.batch.byBatch.indexingStatus.get.mutationOptions(),
)
useEffect(() => {
let timeoutId: ReturnType<typeof setTimeout>
const fetchData = async () => {
await fetchIndexingStatus(
{ params: { dataset_id: datasetId, batch: batchId } },
{
onSuccess: (res) => {
const indexingStatusDetailList = res.data
setIndexingStatusDetail(indexingStatusDetailList)
const isCompleted = indexingStatusDetailList.every((indexingStatusDetail) =>
['completed', 'error', 'paused'].includes(indexingStatusDetail.indexing_status),
)
if (isCompleted) setShouldPoll(false)
},
onSettled: () => {
if (shouldPoll) timeoutId = setTimeout(fetchData, 2500)
},
},
)
}
fetchData()
return () => {
clearTimeout(timeoutId)
}
}, [shouldPoll])
// get rule
const firstDocument = documents[0]
const { data: ruleDetail } = useQuery(
consoleQuery.datasets.processRule.get.queryOptions({
input: { query: { document_id: firstDocument?.id } },
enabled: !!firstDocument,
}),
)
const queryClient = useQueryClient()
const invalidDocumentList = () =>
queryClient.invalidateQueries({
queryKey: consoleQuery.datasets.byDatasetId.documents.get.key(),
})
const documentsHref = `/datasets/${datasetId}/documents`
const apiReferenceUrl = useDatasetApiAccessUrl()
const isEmbeddingWaiting = useMemo(() => {
if (!indexingStatusBatchDetail.length) return false
return indexingStatusBatchDetail.every((indexingStatusDetail) =>
['waiting'].includes(indexingStatusDetail?.indexing_status || ''),
)
}, [indexingStatusBatchDetail])
const isEmbedding = useMemo(() => {
if (!indexingStatusBatchDetail.length) return false
return indexingStatusBatchDetail.some((indexingStatusDetail) =>
['indexing', 'splitting', 'parsing', 'cleaning'].includes(
indexingStatusDetail?.indexing_status || '',
),
)
}, [indexingStatusBatchDetail])
const isEmbeddingCompleted = useMemo(() => {
if (!indexingStatusBatchDetail.length) return false
return indexingStatusBatchDetail.every((indexingStatusDetail) =>
['completed', 'error', 'paused'].includes(indexingStatusDetail?.indexing_status || ''),
)
}, [indexingStatusBatchDetail])
const vectorSpaceAdmissionError = useMemo(
() =>
indexingStatusBatchDetail.find(
(detail) => detail.error_code === 'vector_space_estimate_exceeded',
),
[indexingStatusBatchDetail],
)
const showUpgrade =
deploymentEdition === 'CLOUD' && (plan === 'sandbox' || plan === 'professional')
const getSourceName = (id: string) => {
const doc = documents.find((document) => document.id === id)
return doc?.name
}
const getFileType = (name?: string) => name?.split('.').pop() || 'txt'
const getSourcePercent = (detail: DocumentStatusResponse) => {
const completedCount = detail.completed_segments || 0
const totalCount = detail.total_segments || 0
if (totalCount === 0) return 0
const percent = Math.round((completedCount * 100) / totalCount)
return percent > 100 ? 100 : percent
}
const getSourceType = (id: string) => {
const doc = documents.find((document) => document.id === id)
return doc?.data_source_type
}
const getIcon = (id: string) => {
const doc = documents.find((document) => document.id === id)
const icon = doc?.data_source_info?.notion_page_icon
return typeof icon === 'string' || isNotionPageIcon(icon) ? icon : undefined
}
const isSourceEmbedding = (detail: DocumentStatusResponse) =>
['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'].includes(
detail.indexing_status || '',
)
return (
<>
<div className="flex flex-col gap-y-3">
<div className="flex items-center gap-x-1 system-md-semibold-uppercase text-text-secondary">
{(isEmbeddingWaiting || isEmbedding) && (
<>
<RiLoader2Fill className="size-4 animate-spin" />
<span>
{isEmbeddingWaiting
? t(($) => $['embedding.waiting'], { ns: 'datasetDocuments' })
: t(($) => $['embedding.processing'], { ns: 'datasetDocuments' })}
</span>
</>
)}
{isEmbeddingCompleted && t(($) => $['embedding.completed'], { ns: 'datasetDocuments' })}
</div>
{vectorSpaceAdmissionError?.estimated_vector_space_mb != null &&
vectorSpaceAdmissionError.vector_space_limit_mb != null && (
<VectorSpaceAdmissionAlert
showUpgrade={showUpgrade}
estimatedMb={vectorSpaceAdmissionError.estimated_vector_space_mb}
planLimitMb={vectorSpaceAdmissionError.vector_space_limit_mb}
/>
)}
{deploymentEdition === 'CLOUD' && (plan === 'sandbox' || plan === 'professional') && (
<div className="flex h-13 items-center gap-x-2 rounded-xl border-[0.5px] border-components-panel-border-subtle bg-components-panel-on-panel-item-bg p-2.5 pl-3 shadow-xs shadow-shadow-shadow-3">
<div className="flex shrink-0 items-center justify-center rounded-lg border-[0.5px] border-divider-subtle bg-util-colors-blue-brand-blue-brand-500 shadow-md shadow-shadow-shadow-5">
<RiAedFill className="size-4 text-text-primary-on-surface" />
</div>
<div className="grow system-md-medium text-text-primary">
{t(($) => $['plansCommon.documentProcessingPriorityUpgrade'], { ns: 'billing' })}
</div>
<UpgradeBtn loc="knowledge-speed-up" />
</div>
)}
<div className="flex flex-col gap-0.5 pb-2">
{indexingStatusBatchDetail.map((indexingStatusDetail) => (
<div
key={indexingStatusDetail.id}
className={cn(
'relative h-6.5 overflow-hidden rounded-md bg-components-progress-bar-bg',
indexingStatusDetail.indexing_status === 'error' &&
'bg-state-destructive-hover-alt',
)}
>
{isSourceEmbedding(indexingStatusDetail) && (
<div
className="absolute top-0 left-0 h-full min-w-0.5 border-r-2 border-r-components-progress-bar-progress-highlight bg-components-progress-bar-progress"
style={{ width: `${getSourcePercent(indexingStatusDetail)}%` }}
/>
)}
<div className="z-1 flex h-full items-center gap-1 pr-2 pl-1.5">
{getSourceType(indexingStatusDetail.id) === DatasourceType.localFile && (
<DocumentFileIcon
size="sm"
className="shrink-0"
name={getSourceName(indexingStatusDetail.id)}
extension={getFileType(getSourceName(indexingStatusDetail.id))}
/>
)}
{getSourceType(indexingStatusDetail.id) === DatasourceType.onlineDocument && (
<NotionIcon
className="shrink-0"
type="page"
src={getIcon(indexingStatusDetail.id)}
/>
)}
<div
className="flex w-0 grow items-center gap-1"
title={getSourceName(indexingStatusDetail.id)}
>
<div className="truncate system-xs-medium text-text-secondary">
{getSourceName(indexingStatusDetail.id)}
</div>
<PriorityLabel className="ml-0" />
</div>
{isSourceEmbedding(indexingStatusDetail) && (
<div className="shrink-0 text-xs text-text-secondary">{`${getSourcePercent(indexingStatusDetail)}%`}</div>
)}
{indexingStatusDetail.indexing_status === 'error' && (
<Popover>
<PopoverTrigger
openOnHover
aria-label={indexingStatusDetail.error ?? undefined}
className="inline-flex border-0 bg-transparent p-0"
>
<RiErrorWarningFill className="size-4 shrink-0 text-text-destructive" />
</PopoverTrigger>
<PopoverContent className="max-w-60 rounded-xl border-[0.5px] border-components-panel-border px-4 py-3.5 body-xs-regular text-text-secondary">
{indexingStatusDetail.error}
</PopoverContent>
</Popover>
)}
{indexingStatusDetail.indexing_status === 'completed' && (
<RiCheckboxCircleFill className="size-4 shrink-0 text-text-success" />
)}
</div>
</div>
))}
</div>
<Separator orientation="horizontal" className="my-0 h-[0.5px] bg-divider-subtle" />
<RuleDetail
sourceData={ruleDetail}
indexingType={indexingType}
retrievalMethod={retrievalMethod}
/>
</div>
<div className="mt-6 flex items-center gap-x-2 py-2">
<Link
href={apiReferenceUrl}
target="_blank"
rel="noopener noreferrer"
className={cn(buttonVariants(), 'w-fit')}
>
<RiTerminalBoxLine className="size-4" />
<span>Access the API</span>
</Link>
<Link
href={documentsHref}
className={cn(buttonVariants({ variant: 'primary' }), 'w-fit')}
onClick={invalidDocumentList}
>
<span>{t(($) => $['stepThree.navTo'], { ns: 'datasetCreation' })}</span>
<RiArrowRightLine className="size-4 stroke-current stroke-1" />
</Link>
</div>
</>
)
}
export default EmbeddingProcess
@@ -0,0 +1,100 @@
import type { ProcessRuleResponse } from '@dify/contracts/api/console/datasets/types.gen'
import * as React from 'react'
import { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import { indexMethodIcon, retrievalIcon } from '@/app/components/datasets/create/icons'
import { IndexingType } from '@/app/components/datasets/create/step-two'
import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
import { ProcessMode } from '@/models/datasets'
import { RETRIEVE_METHOD } from '@/types/app'
type RuleDetailProps = {
sourceData?: ProcessRuleResponse
indexingType?: IndexingType
retrievalMethod?: RETRIEVE_METHOD
}
const RuleDetail = ({ sourceData, indexingType, retrievalMethod }: RuleDetailProps) => {
const { t } = useTranslation([
'dataset',
'datasetCreation',
'datasetDocuments',
'datasetSettings',
])
const getValue = useCallback(
(field: string) => {
let value = '-'
switch (field) {
case 'mode':
value = !sourceData?.mode
? value
: sourceData.mode === ProcessMode.general
? (t(($) => $['embedding.custom'], { ns: 'datasetDocuments' }) as string)
: `${t(($) => $['embedding.hierarchical'], { ns: 'datasetDocuments' })} · ${
sourceData?.rules?.parent_mode === 'paragraph'
? t(($) => $['parentMode.paragraph'], { ns: 'dataset' })
: t(($) => $['parentMode.fullDoc'], { ns: 'dataset' })
}`
break
}
return value
},
[sourceData, t],
)
return (
<div className="flex flex-col gap-1" data-testid="rule-detail">
<FieldInfo
label={t(($) => $['embedding.mode'], { ns: 'datasetDocuments' })}
displayedValue={getValue('mode')}
/>
<FieldInfo
label={t(($) => $['stepTwo.indexMode'], { ns: 'datasetCreation' })}
displayedValue={
t(
($) =>
$[`stepTwo.${indexingType === IndexingType.ECONOMICAL ? 'economical' : 'qualified'}`],
{ ns: 'datasetCreation' },
) as string
}
valueIcon={
<img
className="size-4"
src={
indexingType === IndexingType.ECONOMICAL
? indexMethodIcon.economical
: indexMethodIcon.high_quality
}
alt=""
/>
}
/>
<FieldInfo
label={t(($) => $['form.retrievalSetting.title'], { ns: 'datasetSettings' })}
displayedValue={t(
($) =>
$[
`retrieval.${indexingType === IndexingType.ECONOMICAL ? 'keyword_search' : (retrievalMethod ?? 'semantic_search')}.title`
],
{ ns: 'dataset' },
)}
valueIcon={
<img
className="size-4"
src={
retrievalMethod === RETRIEVE_METHOD.fullText
? retrievalIcon.fullText
: retrievalMethod === RETRIEVE_METHOD.hybrid
? retrievalIcon.hybrid
: retrievalIcon.vector
}
alt=""
/>
}
/>
</div>
)
}
export default React.memo(RuleDetail)
@@ -0,0 +1,66 @@
'use client'
import type { PipelineDocument } from '@dify/contracts/api/console/rag/types.gen'
import { RiBookOpenLine } from '@remixicon/react'
import * as React from 'react'
import { useTranslation } from 'react-i18next'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useDocLink } from '@/context/i18n'
import EmbeddingProcess from './embedding-process'
type ProcessingProps = {
batchId: string
documents: PipelineDocument[]
}
const Processing = ({ batchId, documents }: ProcessingProps) => {
const { t } = useTranslation(['datasetCreation', 'datasetPipeline'])
const docLink = useDocLink()
const datasetId = useDatasetDetailContextWithSelector((s) => s.dataset?.id)
const indexingType = useDatasetDetailContextWithSelector(
(s) => s.dataset?.indexing_technique ?? undefined,
)
const retrievalMethod = useDatasetDetailContextWithSelector(
(s) => s.dataset?.retrieval_model_dict?.search_method,
)
return (
<div className="flex size-full justify-center overflow-hidden">
<div className="h-full w-3/5 overflow-y-auto pt-10 pb-8">
<div className="max-w-160">
<EmbeddingProcess
datasetId={datasetId!}
batchId={batchId}
documents={documents}
indexingType={indexingType}
retrievalMethod={retrievalMethod}
/>
</div>
</div>
<div className="w-2/5 pt-22 pr-8">
<div className="flex w-82 flex-col gap-3 rounded-xl bg-background-section p-6">
<div className="flex size-10 items-center justify-center rounded-[10px] bg-components-card-bg shadow-lg shadow-shadow-shadow-5">
<RiBookOpenLine className="size-5 text-text-accent" />
</div>
<div className="flex flex-col gap-y-2">
<div className="system-xl-semibold text-text-secondary">
{t(($) => $['stepThree.sideTipTitle'], { ns: 'datasetCreation' })}
</div>
<div className="system-sm-regular text-text-tertiary">
{t(($) => $['stepThree.sideTipContent'], { ns: 'datasetCreation' })}
</div>
<a
href={docLink('/use-dify/knowledge/knowledge-pipeline/authorize-data-source')}
target="_blank"
rel="noreferrer noopener"
className="system-sm-regular text-text-accent"
>
{t(($) => $['addDocuments.stepThree.learnMore'], { ns: 'datasetPipeline' })}
</a>
</div>
</div>
</div>
</div>
)
}
export default Processing