chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
+204
@@ -0,0 +1,204 @@
|
||||
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import type { ReactElement } from 'react'
|
||||
import { screen, within } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import { renderWithConsoleQuery } from '@/test/console/query-data'
|
||||
import EmbeddingProcess from '../index'
|
||||
|
||||
const status: DocumentStatusResponse = {
|
||||
id: 'document-1',
|
||||
indexing_status: 'completed',
|
||||
completed_segments: 1,
|
||||
total_segments: 1,
|
||||
processing_started_at: null,
|
||||
parsing_completed_at: null,
|
||||
cleaning_completed_at: null,
|
||||
splitting_completed_at: null,
|
||||
completed_at: null,
|
||||
paused_at: null,
|
||||
error: null,
|
||||
stopped_at: null,
|
||||
}
|
||||
|
||||
let deploymentEdition: 'CLOUD' | 'COMMUNITY' = 'COMMUNITY'
|
||||
let mockPlanType: 'sandbox' | 'professional' | 'team' = 'sandbox'
|
||||
let mockPollingState: {
|
||||
statusList: DocumentStatusResponse[]
|
||||
isEmbedding: boolean
|
||||
isEmbeddingCompleted: boolean
|
||||
} = {
|
||||
statusList: [],
|
||||
isEmbedding: false,
|
||||
isEmbeddingCompleted: false,
|
||||
}
|
||||
|
||||
vi.mock('@/hooks/use-api-access-url', () => ({
|
||||
useDatasetApiAccessUrl: () => 'https://api.example.com/docs',
|
||||
}))
|
||||
|
||||
vi.mock('../use-indexing-status-polling', () => ({
|
||||
useIndexingStatusPolling: () => mockPollingState,
|
||||
}))
|
||||
|
||||
vi.mock('../upgrade-banner', () => ({
|
||||
default: () => <div>upgrade processing priority</div>,
|
||||
}))
|
||||
|
||||
function render(ui: ReactElement) {
|
||||
return renderWithConsoleQuery(ui, {
|
||||
systemFeatures: { deployment_edition: deploymentEdition },
|
||||
features: { billing: { subscription: { plan: mockPlanType } } },
|
||||
})
|
||||
}
|
||||
|
||||
describe('EmbeddingProcess', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
deploymentEdition = 'COMMUNITY'
|
||||
mockPlanType = 'sandbox'
|
||||
mockPollingState = {
|
||||
statusList: [],
|
||||
isEmbedding: false,
|
||||
isEmbeddingCompleted: false,
|
||||
}
|
||||
})
|
||||
|
||||
it('shows that document indexing is in progress', () => {
|
||||
mockPollingState.isEmbedding = true
|
||||
|
||||
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
|
||||
|
||||
expect(screen.getByRole('status')).toHaveTextContent('datasetDocuments.embedding.processing')
|
||||
})
|
||||
|
||||
it('shows that document indexing has completed', () => {
|
||||
mockPollingState.isEmbeddingCompleted = true
|
||||
|
||||
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
|
||||
|
||||
expect(screen.getByText('datasetDocuments.embedding.completed')).toHaveAttribute(
|
||||
'role',
|
||||
'status',
|
||||
)
|
||||
})
|
||||
|
||||
it('shows the vector-space admission alert after processing completes', () => {
|
||||
mockPollingState = {
|
||||
statusList: [
|
||||
{
|
||||
...status,
|
||||
id: 'document-1',
|
||||
indexing_status: 'error',
|
||||
error_code: 'vector_space_estimate_exceeded',
|
||||
estimated_vector_space_mb: 61,
|
||||
vector_space_limit_mb: 50,
|
||||
},
|
||||
],
|
||||
isEmbedding: false,
|
||||
isEmbeddingCompleted: true,
|
||||
}
|
||||
|
||||
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
|
||||
|
||||
expect(screen.getByText('datasetDocuments.embedding.completed')).toHaveAttribute(
|
||||
'role',
|
||||
'status',
|
||||
)
|
||||
const alert = screen.getByRole('alert')
|
||||
expect(alert).toHaveTextContent(
|
||||
'datasetDocuments.embedding.vectorSpaceEstimateExceeded.description',
|
||||
)
|
||||
expect(alert).toHaveTextContent('61')
|
||||
expect(alert).toHaveTextContent('50')
|
||||
expect(within(alert).queryByRole('button')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('does not show the vector-space alert for another indexing error', () => {
|
||||
mockPollingState = {
|
||||
statusList: [
|
||||
{
|
||||
...status,
|
||||
id: 'document-1',
|
||||
indexing_status: 'error',
|
||||
error_code: null,
|
||||
estimated_vector_space_mb: 61,
|
||||
vector_space_limit_mb: 50,
|
||||
},
|
||||
],
|
||||
isEmbedding: false,
|
||||
isEmbeddingCompleted: true,
|
||||
}
|
||||
|
||||
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
|
||||
|
||||
expect(screen.queryByRole('alert')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('does not suggest an upgrade to team users', () => {
|
||||
deploymentEdition = 'CLOUD'
|
||||
mockPlanType = 'team'
|
||||
mockPollingState = {
|
||||
statusList: [
|
||||
{
|
||||
...status,
|
||||
id: 'document-1',
|
||||
indexing_status: 'error',
|
||||
error_code: 'vector_space_estimate_exceeded',
|
||||
estimated_vector_space_mb: 61,
|
||||
vector_space_limit_mb: 50,
|
||||
},
|
||||
],
|
||||
isEmbedding: false,
|
||||
isEmbeddingCompleted: true,
|
||||
}
|
||||
|
||||
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
|
||||
|
||||
const alert = screen.getByRole('alert')
|
||||
expect(alert).toHaveTextContent(
|
||||
'datasetDocuments.embedding.vectorSpaceEstimateExceeded.description',
|
||||
)
|
||||
expect(alert).toHaveTextContent('61')
|
||||
expect(alert).toHaveTextContent('50')
|
||||
expect(within(alert).queryByRole('button')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('links to the document list and invalidates its cache on activation', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { queryClient } = render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
|
||||
const documentListKey = consoleQuery.datasets.byDatasetId.documents.get.queryKey({
|
||||
input: { params: { dataset_id: 'dataset-1' } },
|
||||
})
|
||||
queryClient.setQueryData(documentListKey, {
|
||||
data: [],
|
||||
has_more: false,
|
||||
page: 1,
|
||||
limit: 20,
|
||||
total: 0,
|
||||
})
|
||||
|
||||
const link = screen.getByRole('link', { name: 'datasetCreation.stepThree.navTo' })
|
||||
expect(link).toHaveAttribute('href', '/datasets/dataset-1/documents')
|
||||
await user.click(link)
|
||||
|
||||
expect(queryClient.getQueryState(documentListKey)?.isInvalidated).toBe(true)
|
||||
})
|
||||
|
||||
it('links to the dataset API reference', () => {
|
||||
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
|
||||
|
||||
expect(screen.getByRole('link', { name: 'Access the API' })).toHaveAttribute(
|
||||
'href',
|
||||
'https://api.example.com/docs',
|
||||
)
|
||||
})
|
||||
|
||||
it('offers a processing-priority upgrade outside the team plan', () => {
|
||||
deploymentEdition = 'CLOUD'
|
||||
|
||||
render(<EmbeddingProcess datasetId="dataset-1" batchId="batch-1" />)
|
||||
|
||||
expect(screen.getByText('upgrade processing priority')).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vite-plus/test'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import IndexingProgressItem from '../indexing-progress-item'
|
||||
|
||||
// Billing labels have their own tests and are outside indexing progress semantics.
|
||||
vi.mock('@/app/components/billing/priority-label', () => ({
|
||||
default: () => <span data-testid="priority-label">Priority</span>,
|
||||
}))
|
||||
describe('IndexingProgressItem', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
const makeDetail = (overrides: Partial<DocumentStatusResponse> = {}): DocumentStatusResponse => ({
|
||||
id: 'doc-1',
|
||||
indexing_status: 'indexing',
|
||||
processing_started_at: 0,
|
||||
parsing_completed_at: 0,
|
||||
cleaning_completed_at: 0,
|
||||
splitting_completed_at: 0,
|
||||
completed_at: null,
|
||||
paused_at: null,
|
||||
error: null,
|
||||
stopped_at: null,
|
||||
completed_segments: 50,
|
||||
total_segments: 100,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
it('should render name and progress for embedding status', () => {
|
||||
render(
|
||||
<IndexingProgressItem
|
||||
detail={makeDetail()}
|
||||
name="test.pdf"
|
||||
sourceType={DataSourceType.FILE}
|
||||
/>,
|
||||
)
|
||||
|
||||
expect(screen.getByText('test.pdf')).toBeInTheDocument()
|
||||
expect(screen.getByText('50%')).toBeInTheDocument()
|
||||
expect(screen.getByRole('progressbar', { name: 'test.pdf' })).toHaveAttribute(
|
||||
'aria-valuenow',
|
||||
'50',
|
||||
)
|
||||
})
|
||||
|
||||
it('announces a file completing after processing without retaining its progress bar', () => {
|
||||
const { rerender } = render(<IndexingProgressItem detail={makeDetail()} name="done.pdf" />)
|
||||
const status = screen.getByRole('status')
|
||||
expect(status).toBeEmptyDOMElement()
|
||||
expect(screen.getByRole('progressbar', { name: 'done.pdf' })).toBeInTheDocument()
|
||||
|
||||
rerender(
|
||||
<IndexingProgressItem
|
||||
detail={makeDetail({ indexing_status: 'completed' })}
|
||||
name="done.pdf"
|
||||
/>,
|
||||
)
|
||||
|
||||
expect(status).toHaveTextContent('done.pdf: datasetDocuments.embedding.completed')
|
||||
expect(screen.queryByRole('progressbar')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('opens the full indexing error from the warning button', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(
|
||||
<IndexingProgressItem
|
||||
detail={makeDetail({ indexing_status: 'error', error: 'Parse failed' })}
|
||||
name="broken.pdf"
|
||||
/>,
|
||||
)
|
||||
|
||||
expect(screen.getByRole('status')).toHaveTextContent('broken.pdf: common.error: Parse failed')
|
||||
await user.click(screen.getByRole('button', { name: 'common.error' }))
|
||||
expect(await screen.findByRole('dialog')).toHaveTextContent('Parse failed')
|
||||
await user.keyboard('{Escape}')
|
||||
expect(screen.getByRole('button', { name: 'common.error' })).toHaveFocus()
|
||||
})
|
||||
|
||||
it('should use the localized fallback when an error has no message', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(
|
||||
<IndexingProgressItem
|
||||
detail={makeDetail({ indexing_status: 'error', error: null })}
|
||||
name="broken.pdf"
|
||||
/>,
|
||||
)
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'common.error' }))
|
||||
expect(await screen.findByRole('dialog')).toHaveTextContent('common.error')
|
||||
})
|
||||
})
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
import type { ProcessRuleResponse } from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import { ProcessMode } from '@/models/datasets'
|
||||
import { RETRIEVE_METHOD } from '@/types/app'
|
||||
import RuleDetail from '../rule-detail'
|
||||
|
||||
const rule: ProcessRuleResponse = {
|
||||
limits: { indexing_max_segmentation_tokens_length: 4000 },
|
||||
mode: ProcessMode.general,
|
||||
rules: {
|
||||
segmentation: { separator: '\n', max_tokens: 500, chunk_overlap: 50 },
|
||||
pre_processing_rules: [
|
||||
{ id: 'remove_extra_spaces', enabled: true },
|
||||
{ id: 'remove_urls_emails', enabled: false },
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
it('summarizes the custom segmentation and only enabled cleaning rules', () => {
|
||||
render(
|
||||
<RuleDetail
|
||||
sourceData={rule}
|
||||
indexingType="high_quality"
|
||||
retrievalMethod={RETRIEVE_METHOD.semantic}
|
||||
/>,
|
||||
)
|
||||
expect(screen.getByText('datasetDocuments.embedding.custom')).toBeInTheDocument()
|
||||
expect(screen.getByText('500')).toBeInTheDocument()
|
||||
expect(screen.getByText('datasetCreation.stepTwo.removeExtraSpaces')).toBeInTheDocument()
|
||||
expect(screen.queryByText('datasetCreation.stepTwo.removeUrlEmails')).not.toBeInTheDocument()
|
||||
expect(screen.getByText('datasetCreation.stepTwo.qualified')).toBeInTheDocument()
|
||||
expect(screen.getByText('dataset.retrieval.semantic_search.title')).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('shows parent and child chunk limits for hierarchical processing', () => {
|
||||
render(
|
||||
<RuleDetail
|
||||
sourceData={{
|
||||
...rule,
|
||||
mode: ProcessMode.parentChild,
|
||||
rules: {
|
||||
parent_mode: 'paragraph',
|
||||
segmentation: { separator: '\n', max_tokens: 1000 },
|
||||
subchunk_segmentation: { separator: '\n', max_tokens: 200 },
|
||||
},
|
||||
}}
|
||||
indexingType="high_quality"
|
||||
/>,
|
||||
)
|
||||
expect(
|
||||
screen.getByText('datasetDocuments.embedding.hierarchical · dataset.parentMode.paragraph'),
|
||||
).toBeInTheDocument()
|
||||
expect(
|
||||
screen.getByText(
|
||||
'datasetDocuments.embedding.parentMaxTokens 1000; datasetDocuments.embedding.childMaxTokens 200',
|
||||
),
|
||||
).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('uses keyword retrieval for economical indexing', () => {
|
||||
render(
|
||||
<RuleDetail
|
||||
sourceData={rule}
|
||||
indexingType="economy"
|
||||
retrievalMethod={RETRIEVE_METHOD.semantic}
|
||||
/>,
|
||||
)
|
||||
expect(screen.getByText('datasetCreation.stepTwo.economical')).toBeInTheDocument()
|
||||
expect(screen.getByText('dataset.retrieval.keyword_search.title')).toBeInTheDocument()
|
||||
expect(screen.queryByText('dataset.retrieval.semantic_search.title')).not.toBeInTheDocument()
|
||||
})
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import { act, renderHook } from '@testing-library/react'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
|
||||
import { useIndexingStatusPolling } from '../use-indexing-status-polling'
|
||||
|
||||
const mockFetchIndexingStatusBatch = vi.fn()
|
||||
|
||||
vi.mock('@/service/datasets', () => ({
|
||||
fetchIndexingStatusBatch: (...args: unknown[]) => mockFetchIndexingStatusBatch(...args),
|
||||
}))
|
||||
|
||||
const status = (
|
||||
indexing_status: DocumentStatusResponse['indexing_status'],
|
||||
overrides: Partial<DocumentStatusResponse> = {},
|
||||
): DocumentStatusResponse => ({
|
||||
id: 'doc-1',
|
||||
indexing_status,
|
||||
processing_started_at: null,
|
||||
parsing_completed_at: null,
|
||||
cleaning_completed_at: null,
|
||||
splitting_completed_at: null,
|
||||
completed_at: null,
|
||||
paused_at: null,
|
||||
error: null,
|
||||
stopped_at: null,
|
||||
completed_segments: 0,
|
||||
total_segments: 0,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
describe('useIndexingStatusPolling', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
const defaultParams = { datasetId: 'ds-1', batchId: 'batch-1' }
|
||||
|
||||
it('should fetch status on mount and update state', async () => {
|
||||
mockFetchIndexingStatusBatch.mockResolvedValue({
|
||||
data: [status('indexing', { completed_segments: 5, total_segments: 10 })],
|
||||
})
|
||||
|
||||
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
|
||||
// Flush the resolved promise
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
|
||||
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledWith({
|
||||
datasetId: 'ds-1',
|
||||
batchId: 'batch-1',
|
||||
})
|
||||
expect(result.current.statusList).toHaveLength(1)
|
||||
expect(result.current.isEmbedding).toBe(true)
|
||||
expect(result.current.isEmbeddingCompleted).toBe(false)
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(2499)
|
||||
})
|
||||
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(1)
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
})
|
||||
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('should stop polling when all completed', async () => {
|
||||
mockFetchIndexingStatusBatch.mockResolvedValue({
|
||||
data: [status('completed')],
|
||||
})
|
||||
|
||||
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
|
||||
expect(result.current.isEmbeddingCompleted).toBe(true)
|
||||
expect(result.current.isEmbedding).toBe(false)
|
||||
|
||||
// Should not schedule another poll
|
||||
const callCount = mockFetchIndexingStatusBatch.mock.calls.length
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(5000)
|
||||
})
|
||||
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(callCount)
|
||||
})
|
||||
|
||||
it('should continue polling on fetch error', async () => {
|
||||
mockFetchIndexingStatusBatch.mockRejectedValueOnce(new Error('network')).mockResolvedValueOnce({
|
||||
data: [status('completed')],
|
||||
})
|
||||
|
||||
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
|
||||
// First call: rejects
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
// Advance past polling interval for retry
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(2500)
|
||||
})
|
||||
|
||||
expect(result.current.isEmbeddingCompleted).toBe(true)
|
||||
})
|
||||
|
||||
it('should detect embedding statuses', async () => {
|
||||
mockFetchIndexingStatusBatch.mockResolvedValue({
|
||||
data: [status('splitting'), status('parsing')],
|
||||
})
|
||||
|
||||
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
|
||||
expect(result.current.isEmbedding).toBe(true)
|
||||
expect(result.current.isEmbeddingCompleted).toBe(false)
|
||||
})
|
||||
|
||||
it('should detect mixed statuses (some completed, some embedding)', async () => {
|
||||
mockFetchIndexingStatusBatch.mockResolvedValue({
|
||||
data: [status('completed'), status('indexing')],
|
||||
})
|
||||
|
||||
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
|
||||
expect(result.current.statusList).toHaveLength(2)
|
||||
expect(result.current.isEmbedding).toBe(true)
|
||||
expect(result.current.isEmbeddingCompleted).toBe(false)
|
||||
})
|
||||
|
||||
it('should cleanup on unmount', async () => {
|
||||
mockFetchIndexingStatusBatch.mockResolvedValue({
|
||||
data: [status('indexing')],
|
||||
})
|
||||
|
||||
const { unmount } = renderHook(() => useIndexingStatusPolling(defaultParams))
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
|
||||
const callCount = mockFetchIndexingStatusBatch.mock.calls.length
|
||||
unmount()
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(5000)
|
||||
})
|
||||
expect(mockFetchIndexingStatusBatch).toHaveBeenCalledTimes(callCount)
|
||||
})
|
||||
|
||||
it('should treat error and paused as completed statuses', async () => {
|
||||
mockFetchIndexingStatusBatch.mockResolvedValue({
|
||||
data: [status('error'), status('paused')],
|
||||
})
|
||||
|
||||
const { result } = renderHook(() => useIndexingStatusPolling(defaultParams))
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
})
|
||||
|
||||
expect(result.current.isEmbeddingCompleted).toBe(true)
|
||||
expect(result.current.isEmbedding).toBe(false)
|
||||
})
|
||||
})
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
import type {
|
||||
DocumentResponse,
|
||||
DocumentStatusResponse,
|
||||
} from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import { describe, expect, it } from 'vite-plus/test'
|
||||
import { createDocumentLookup, getFileType, getSourcePercent, isSourceEmbedding } from '../utils'
|
||||
|
||||
const status = (overrides: Partial<DocumentStatusResponse> = {}): DocumentStatusResponse => ({
|
||||
id: 'doc-1',
|
||||
indexing_status: 'indexing',
|
||||
processing_started_at: null,
|
||||
parsing_completed_at: null,
|
||||
cleaning_completed_at: null,
|
||||
splitting_completed_at: null,
|
||||
completed_at: null,
|
||||
paused_at: null,
|
||||
error: null,
|
||||
stopped_at: null,
|
||||
completed_segments: 0,
|
||||
total_segments: 0,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
describe('isSourceEmbedding', () => {
|
||||
const embeddingStatuses = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const
|
||||
const nonEmbeddingStatuses = ['completed', 'error', 'paused'] as const
|
||||
|
||||
it.each(embeddingStatuses)('should return true for status "%s"', (indexing_status) => {
|
||||
expect(isSourceEmbedding(status({ indexing_status }))).toBe(true)
|
||||
})
|
||||
|
||||
it.each(nonEmbeddingStatuses)('should return false for status "%s"', (indexing_status) => {
|
||||
expect(isSourceEmbedding(status({ indexing_status }))).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getSourcePercent', () => {
|
||||
it('should calculate correct percentage', () => {
|
||||
expect(getSourcePercent(status({ completed_segments: 50, total_segments: 100 }))).toBe(50)
|
||||
})
|
||||
|
||||
it('should return 0 when total is 0', () => {
|
||||
expect(getSourcePercent(status({ completed_segments: 0, total_segments: 0 }))).toBe(0)
|
||||
})
|
||||
|
||||
it('should cap at 100', () => {
|
||||
expect(getSourcePercent(status({ completed_segments: 150, total_segments: 100 }))).toBe(100)
|
||||
})
|
||||
|
||||
it('should round to nearest integer', () => {
|
||||
expect(getSourcePercent(status({ completed_segments: 1, total_segments: 3 }))).toBe(33)
|
||||
})
|
||||
|
||||
it('treats absent segment totals as zero', () => {
|
||||
expect(getSourcePercent(status({ completed_segments: null, total_segments: null }))).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getFileType', () => {
|
||||
it('should extract extension from filename', () => {
|
||||
expect(getFileType('document.pdf')).toBe('pdf')
|
||||
})
|
||||
|
||||
it('should return last extension for multi-dot names', () => {
|
||||
expect(getFileType('archive.tar.gz')).toBe('gz')
|
||||
})
|
||||
|
||||
it('should default to "txt" for undefined', () => {
|
||||
expect(getFileType(undefined)).toBe('txt')
|
||||
})
|
||||
|
||||
it('should default to "txt" for empty string', () => {
|
||||
expect(getFileType('')).toBe('txt')
|
||||
})
|
||||
})
|
||||
|
||||
describe('createDocumentLookup', () => {
|
||||
const documents: DocumentResponse[] = [
|
||||
{
|
||||
data_source_detail_dict: {},
|
||||
id: 'doc-1',
|
||||
name: 'test.pdf',
|
||||
data_source_type: 'upload_file',
|
||||
data_source_info: {
|
||||
upload_file: { id: 'f1', name: 'test.pdf' },
|
||||
notion_page_icon: undefined,
|
||||
},
|
||||
},
|
||||
{
|
||||
data_source_detail_dict: {},
|
||||
id: 'doc-2',
|
||||
name: 'notion-page',
|
||||
data_source_type: 'notion_import',
|
||||
data_source_info: {
|
||||
upload_file: { id: 'f2', name: '' },
|
||||
notion_page_icon: 'https://icon.url',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
it('should get name by id', () => {
|
||||
const lookup = createDocumentLookup(documents)
|
||||
expect(lookup.getName('doc-1')).toBe('test.pdf')
|
||||
})
|
||||
|
||||
it('should get source type by id', () => {
|
||||
const lookup = createDocumentLookup(documents)
|
||||
expect(lookup.getSourceType('doc-1')).toBe('upload_file')
|
||||
})
|
||||
|
||||
it('should get notion icon for legacy data source', () => {
|
||||
const lookup = createDocumentLookup(documents)
|
||||
expect(lookup.getNotionIcon('doc-2')).toBe('https://icon.url')
|
||||
})
|
||||
|
||||
it('should return undefined notion icon for non-legacy info', () => {
|
||||
const docs: DocumentResponse[] = [
|
||||
{
|
||||
id: 'doc-3',
|
||||
name: 'New source',
|
||||
data_source_detail_dict: {},
|
||||
data_source_info: { some_other: 'field' },
|
||||
},
|
||||
]
|
||||
const lookup = createDocumentLookup(docs)
|
||||
expect(lookup.getNotionIcon('doc-3')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('should handle empty documents list', () => {
|
||||
const lookup = createDocumentLookup([])
|
||||
expect(lookup.getDocument('any')).toBeUndefined()
|
||||
expect(lookup.getName('any')).toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,181 @@
|
||||
import type { DocumentResponse } from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import type { FC } from 'react'
|
||||
import type { RETRIEVE_METHOD } from '@/types/app'
|
||||
import { buttonVariants } from '@langgenius/dify-ui/button'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { Separator } from '@langgenius/dify-ui/separator'
|
||||
import { RiArrowRightLine, RiLoader2Fill, RiTerminalBoxLine } from '@remixicon/react'
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query'
|
||||
import { useAtomValue } from 'jotai'
|
||||
import { useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import VectorSpaceAdmissionAlert from '@/app/components/datasets/common/vector-space-admission-alert'
|
||||
import { deploymentEditionAtom } from '@/features/system-features/state'
|
||||
import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url'
|
||||
import Link from '@/next/link'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import IndexingProgressItem from './indexing-progress-item'
|
||||
import RuleDetail from './rule-detail'
|
||||
import UpgradeBanner from './upgrade-banner'
|
||||
import { useIndexingStatusPolling } from './use-indexing-status-polling'
|
||||
import { createDocumentLookup } from './utils'
|
||||
|
||||
type EmbeddingProcessProps = {
|
||||
datasetId: string
|
||||
batchId: string
|
||||
documents?: DocumentResponse[]
|
||||
indexingType?: string
|
||||
retrievalMethod?: RETRIEVE_METHOD
|
||||
}
|
||||
|
||||
// Status header component
|
||||
const StatusHeader: FC<{ isEmbedding: boolean; isCompleted: boolean }> = ({
|
||||
isEmbedding,
|
||||
isCompleted,
|
||||
}) => {
|
||||
const { t } = useTranslation(['datasetDocuments'])
|
||||
|
||||
return (
|
||||
<div
|
||||
role="status"
|
||||
aria-atomic="true"
|
||||
className="flex items-center gap-x-1 system-md-semibold-uppercase text-text-secondary"
|
||||
>
|
||||
{isEmbedding && (
|
||||
<>
|
||||
<RiLoader2Fill aria-hidden="true" className="size-4 animate-spin" />
|
||||
<span>{t(($) => $['embedding.processing'], { ns: 'datasetDocuments' })}</span>
|
||||
</>
|
||||
)}
|
||||
{isCompleted && t(($) => $['embedding.completed'], { ns: 'datasetDocuments' })}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Action buttons component
|
||||
const ActionButtons: FC<{
|
||||
apiReferenceUrl: string
|
||||
documentsHref: string
|
||||
onNavigateToDocuments: () => void
|
||||
}> = ({ apiReferenceUrl, documentsHref, onNavigateToDocuments }) => {
|
||||
const { t } = useTranslation(['datasetCreation'])
|
||||
|
||||
return (
|
||||
<div className="mt-6 flex flex-wrap items-center gap-2 py-2">
|
||||
<Link
|
||||
href={apiReferenceUrl}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className={cn(buttonVariants(), 'w-fit')}
|
||||
>
|
||||
<RiTerminalBoxLine className="size-4" />
|
||||
<span>Access the API</span>
|
||||
</Link>
|
||||
<Link
|
||||
href={documentsHref}
|
||||
className={cn(buttonVariants({ variant: 'primary' }), 'w-fit')}
|
||||
onClick={onNavigateToDocuments}
|
||||
>
|
||||
<span>{t(($) => $['stepThree.navTo'], { ns: 'datasetCreation' })}</span>
|
||||
<RiArrowRightLine className="size-4 stroke-current stroke-1" />
|
||||
</Link>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const EmbeddingProcess: FC<EmbeddingProcessProps> = ({
|
||||
datasetId,
|
||||
batchId,
|
||||
documents = [],
|
||||
indexingType,
|
||||
retrievalMethod,
|
||||
}) => {
|
||||
const deploymentEdition = useAtomValue(deploymentEditionAtom)
|
||||
const { data: plan } = useQuery(
|
||||
consoleQuery.features.get.queryOptions({
|
||||
enabled: deploymentEdition === 'CLOUD',
|
||||
select: (data) => data.billing.subscription.plan,
|
||||
}),
|
||||
)
|
||||
const queryClient = useQueryClient()
|
||||
const invalidDocumentList = () =>
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: consoleQuery.datasets.byDatasetId.documents.get.key(),
|
||||
})
|
||||
const apiReferenceUrl = useDatasetApiAccessUrl()
|
||||
|
||||
// Polling hook for indexing status
|
||||
const { statusList, isEmbedding, isEmbeddingCompleted } = useIndexingStatusPolling({
|
||||
datasetId,
|
||||
batchId,
|
||||
})
|
||||
|
||||
// Get process rule for the first document
|
||||
const firstDocumentId = documents[0]?.id
|
||||
const { data: ruleDetail } = useQuery(
|
||||
consoleQuery.datasets.processRule.get.queryOptions({
|
||||
input: { query: { document_id: firstDocumentId } },
|
||||
enabled: !!firstDocumentId,
|
||||
}),
|
||||
)
|
||||
|
||||
// Document lookup utilities - memoized for performance
|
||||
const documentLookup = useMemo(() => createDocumentLookup(documents), [documents])
|
||||
|
||||
const documentsHref = `/datasets/${datasetId}/documents`
|
||||
|
||||
const showUpgradeBanner =
|
||||
deploymentEdition === 'CLOUD' && (plan === 'sandbox' || plan === 'professional')
|
||||
const showVectorSpaceUpgrade =
|
||||
deploymentEdition === 'CLOUD' && (plan === 'sandbox' || plan === 'professional')
|
||||
const vectorSpaceAdmissionError = statusList.find(
|
||||
(detail) => detail.error_code === 'vector_space_estimate_exceeded',
|
||||
)
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="flex flex-col gap-y-3">
|
||||
<StatusHeader isEmbedding={isEmbedding} isCompleted={isEmbeddingCompleted} />
|
||||
|
||||
{vectorSpaceAdmissionError?.estimated_vector_space_mb != null &&
|
||||
vectorSpaceAdmissionError.vector_space_limit_mb != null && (
|
||||
<VectorSpaceAdmissionAlert
|
||||
showUpgrade={showVectorSpaceUpgrade}
|
||||
estimatedMb={vectorSpaceAdmissionError.estimated_vector_space_mb}
|
||||
planLimitMb={vectorSpaceAdmissionError.vector_space_limit_mb}
|
||||
/>
|
||||
)}
|
||||
|
||||
{showUpgradeBanner && <UpgradeBanner />}
|
||||
|
||||
<div className="flex flex-col gap-0.5 pb-2">
|
||||
{statusList.map((detail) => (
|
||||
<IndexingProgressItem
|
||||
key={detail.id}
|
||||
detail={detail}
|
||||
name={documentLookup.getName(detail.id)}
|
||||
sourceType={documentLookup.getSourceType(detail.id)}
|
||||
notionIcon={documentLookup.getNotionIcon(detail.id)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<Separator orientation="horizontal" className="my-0 h-[0.5px] bg-divider-subtle" />
|
||||
|
||||
<RuleDetail
|
||||
sourceData={ruleDetail}
|
||||
indexingType={indexingType}
|
||||
retrievalMethod={retrievalMethod}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<ActionButtons
|
||||
apiReferenceUrl={apiReferenceUrl}
|
||||
documentsHref={documentsHref}
|
||||
onNavigateToDocuments={invalidDocumentList}
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default EmbeddingProcess
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import type { ComponentProps, FC } from 'react'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { Infotip, InfotipContent, InfotipTrigger } from '@langgenius/dify-ui/infotip'
|
||||
import { RiCheckboxCircleFill } from '@remixicon/react'
|
||||
import { useId } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import NotionIcon from '@/app/components/base/notion-icon'
|
||||
import PriorityLabel from '@/app/components/billing/priority-label'
|
||||
import { DataSourceType } from '@/models/datasets'
|
||||
import DocumentFileIcon from '../../common/document-file-icon'
|
||||
import { getFileType, getSourcePercent, isSourceEmbedding } from './utils'
|
||||
|
||||
type IndexingProgressItemProps = {
|
||||
detail: DocumentStatusResponse
|
||||
name?: string
|
||||
sourceType?: DataSourceType
|
||||
notionIcon?: ComponentProps<typeof NotionIcon>['src']
|
||||
}
|
||||
|
||||
// Status icon component for completed/error states
|
||||
const StatusIcon: FC<{ status: string; error?: string }> = ({ status, error }) => {
|
||||
const { t } = useTranslation(['common'])
|
||||
|
||||
if (status === 'completed')
|
||||
return <RiCheckboxCircleFill aria-hidden="true" className="size-4 shrink-0 text-text-success" />
|
||||
|
||||
if (status === 'error') {
|
||||
const errorLabel = error || t(($) => $.error, { ns: 'common' })
|
||||
|
||||
return (
|
||||
<Infotip>
|
||||
<InfotipTrigger
|
||||
aria-label={t(($) => $.error, { ns: 'common' })}
|
||||
iconVariant="warning"
|
||||
iconSize="large"
|
||||
className="text-text-destructive"
|
||||
/>
|
||||
<InfotipContent
|
||||
aria-label={t(($) => $.error, { ns: 'common' })}
|
||||
className="whitespace-pre-wrap"
|
||||
sideOffset={4}
|
||||
>
|
||||
{errorLabel}
|
||||
</InfotipContent>
|
||||
</Infotip>
|
||||
)
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
// Source type icon component
|
||||
const SourceTypeIcon: FC<{
|
||||
sourceType?: DataSourceType
|
||||
name?: string
|
||||
notionIcon?: ComponentProps<typeof NotionIcon>['src']
|
||||
}> = ({ sourceType, name, notionIcon }) => {
|
||||
if (sourceType === DataSourceType.FILE) {
|
||||
return (
|
||||
<DocumentFileIcon size="sm" className="shrink-0" name={name} extension={getFileType(name)} />
|
||||
)
|
||||
}
|
||||
|
||||
if (sourceType === DataSourceType.NOTION) {
|
||||
return <NotionIcon className="shrink-0" type="page" src={notionIcon} />
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
const IndexingProgressItem: FC<IndexingProgressItemProps> = ({
|
||||
detail,
|
||||
name,
|
||||
sourceType,
|
||||
notionIcon,
|
||||
}) => {
|
||||
const { t } = useTranslation(['common', 'datasetDocuments'])
|
||||
const nameId = useId()
|
||||
const isEmbedding = isSourceEmbedding(detail)
|
||||
const percent = getSourcePercent(detail)
|
||||
const isError = detail.indexing_status === 'error'
|
||||
const statusText =
|
||||
detail.indexing_status === 'completed'
|
||||
? t(($) => $['embedding.completed'], { ns: 'datasetDocuments' })
|
||||
: isError
|
||||
? `${t(($) => $.error, { ns: 'common' })}: ${detail.error || ''}`
|
||||
: detail.indexing_status === 'paused'
|
||||
? t(($) => $['embedding.paused'], { ns: 'datasetDocuments' })
|
||||
: ''
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
'relative h-6.5 overflow-hidden rounded-md bg-components-progress-bar-bg',
|
||||
isError && 'bg-state-destructive-hover-alt',
|
||||
)}
|
||||
>
|
||||
{isEmbedding && (
|
||||
<div
|
||||
className="absolute top-0 left-0 h-full min-w-0.5 border-r-2 border-r-components-progress-bar-progress-highlight bg-components-progress-bar-progress"
|
||||
role="progressbar"
|
||||
aria-labelledby={nameId}
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-valuenow={percent}
|
||||
style={{ width: `${percent}%` }}
|
||||
/>
|
||||
)}
|
||||
<span role="status" aria-atomic="true" className="sr-only">
|
||||
{statusText && `${name}: ${statusText}`}
|
||||
</span>
|
||||
<div className="z-1 flex h-full items-center gap-1 pr-2 pl-1.5">
|
||||
<SourceTypeIcon sourceType={sourceType} name={name} notionIcon={notionIcon} />
|
||||
<div className="flex w-0 grow items-center gap-1" title={name}>
|
||||
<div id={nameId} className="truncate system-xs-medium text-text-secondary">
|
||||
{name}
|
||||
</div>
|
||||
<PriorityLabel className="ml-0" />
|
||||
</div>
|
||||
{isEmbedding && <div className="shrink-0 text-xs text-text-secondary">{`${percent}%`}</div>}
|
||||
<StatusIcon status={detail.indexing_status} error={detail.error ?? undefined} />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default IndexingProgressItem
|
||||
@@ -0,0 +1,146 @@
|
||||
import type { ProcessRuleResponse } from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import type { FC } from 'react'
|
||||
import { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
|
||||
import { ProcessMode } from '@/models/datasets'
|
||||
import { RETRIEVE_METHOD } from '@/types/app'
|
||||
import { indexMethodIcon, retrievalIcon } from '../icons'
|
||||
import { IndexingType } from '../step-two'
|
||||
|
||||
type RuleDetailProps = {
|
||||
sourceData?: ProcessRuleResponse
|
||||
indexingType?: string
|
||||
retrievalMethod?: RETRIEVE_METHOD
|
||||
}
|
||||
|
||||
// Lookup table for pre-processing rule names
|
||||
const PRE_PROCESSING_RULE_KEYS = {
|
||||
remove_extra_spaces: 'stepTwo.removeExtraSpaces',
|
||||
remove_urls_emails: 'stepTwo.removeUrlEmails',
|
||||
remove_stopwords: 'stepTwo.removeStopwords',
|
||||
} as const
|
||||
|
||||
// Lookup table for retrieval method icons
|
||||
const RETRIEVAL_ICON_MAP: Partial<Record<RETRIEVE_METHOD, string>> = {
|
||||
[RETRIEVE_METHOD.fullText]: retrievalIcon.fullText,
|
||||
[RETRIEVE_METHOD.hybrid]: retrievalIcon.hybrid,
|
||||
[RETRIEVE_METHOD.semantic]: retrievalIcon.vector,
|
||||
[RETRIEVE_METHOD.keywordSearch]: retrievalIcon.fullText,
|
||||
}
|
||||
|
||||
const isNumber = (value: unknown): value is number => typeof value === 'number'
|
||||
|
||||
const RuleDetail: FC<RuleDetailProps> = ({ sourceData, indexingType, retrievalMethod }) => {
|
||||
const { t } = useTranslation([
|
||||
'dataset',
|
||||
'datasetCreation',
|
||||
'datasetDocuments',
|
||||
'datasetSettings',
|
||||
])
|
||||
|
||||
const segmentationRuleLabels = {
|
||||
mode: t(($) => $['embedding.mode'], { ns: 'datasetDocuments' }),
|
||||
segmentLength: t(($) => $['embedding.segmentLength'], { ns: 'datasetDocuments' }),
|
||||
textCleaning: t(($) => $['embedding.textCleaning'], { ns: 'datasetDocuments' }),
|
||||
}
|
||||
|
||||
const getRuleName = useCallback(
|
||||
(key: string): string | undefined => {
|
||||
const translationKey = PRE_PROCESSING_RULE_KEYS[key as keyof typeof PRE_PROCESSING_RULE_KEYS]
|
||||
return translationKey ? t(($) => $[translationKey], { ns: 'datasetCreation' }) : undefined
|
||||
},
|
||||
[t],
|
||||
)
|
||||
|
||||
const getModeValue = useCallback((): string => {
|
||||
if (!sourceData?.mode) return '-'
|
||||
|
||||
if (sourceData.mode === ProcessMode.general)
|
||||
return t(($) => $['embedding.custom'], { ns: 'datasetDocuments' })
|
||||
|
||||
const parentModeLabel =
|
||||
sourceData.rules?.parent_mode === 'paragraph'
|
||||
? t(($) => $['parentMode.paragraph'], { ns: 'dataset' })
|
||||
: t(($) => $['parentMode.fullDoc'], { ns: 'dataset' })
|
||||
|
||||
return `${t(($) => $['embedding.hierarchical'], { ns: 'datasetDocuments' })} · ${parentModeLabel}`
|
||||
}, [sourceData, t])
|
||||
|
||||
const getSegmentLengthValue = useCallback((): string | number => {
|
||||
if (!sourceData?.mode) return '-'
|
||||
|
||||
const maxTokens = isNumber(sourceData.rules?.segmentation?.max_tokens)
|
||||
? sourceData.rules.segmentation.max_tokens
|
||||
: '-'
|
||||
|
||||
if (sourceData.mode === ProcessMode.general) return maxTokens
|
||||
|
||||
const childMaxTokens = isNumber(sourceData.rules?.subchunk_segmentation?.max_tokens)
|
||||
? sourceData.rules.subchunk_segmentation.max_tokens
|
||||
: '-'
|
||||
|
||||
return `${t(($) => $['embedding.parentMaxTokens'], { ns: 'datasetDocuments' })} ${maxTokens}; ${t(($) => $['embedding.childMaxTokens'], { ns: 'datasetDocuments' })} ${childMaxTokens}`
|
||||
}, [sourceData, t])
|
||||
|
||||
const getTextCleaningValue = useCallback((): string => {
|
||||
if (!sourceData?.mode) return '-'
|
||||
|
||||
const enabledRules =
|
||||
sourceData.rules?.pre_processing_rules?.filter((rule) => rule.enabled) || []
|
||||
const ruleNames = enabledRules
|
||||
.map((rule) => {
|
||||
const name = getRuleName(rule.id)
|
||||
return typeof name === 'string' ? name : ''
|
||||
})
|
||||
.filter((name) => name)
|
||||
return ruleNames.length > 0 ? ruleNames.join(',') : '-'
|
||||
}, [sourceData, getRuleName])
|
||||
|
||||
const fieldValueGetters: Record<string, () => string | number> = {
|
||||
mode: getModeValue,
|
||||
segmentLength: getSegmentLengthValue,
|
||||
textCleaning: getTextCleaningValue,
|
||||
}
|
||||
|
||||
const isEconomical = indexingType === IndexingType.ECONOMICAL
|
||||
const indexMethodIconSrc = isEconomical
|
||||
? indexMethodIcon.economical
|
||||
: indexMethodIcon.high_quality
|
||||
const indexModeLabel = t(($) => $[`stepTwo.${isEconomical ? 'economical' : 'qualified'}`], {
|
||||
ns: 'datasetCreation',
|
||||
})
|
||||
|
||||
const effectiveRetrievalMethod = isEconomical
|
||||
? 'keyword_search'
|
||||
: (retrievalMethod ?? 'semantic_search')
|
||||
const retrievalLabel = t(($) => $[`retrieval.${effectiveRetrievalMethod}.title`], {
|
||||
ns: 'dataset',
|
||||
})
|
||||
const retrievalIconSrc =
|
||||
RETRIEVAL_ICON_MAP[retrievalMethod as keyof typeof RETRIEVAL_ICON_MAP] ?? retrievalIcon.vector
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1">
|
||||
{Object.keys(segmentationRuleLabels).map((field) => (
|
||||
<FieldInfo
|
||||
key={field}
|
||||
label={segmentationRuleLabels[field as keyof typeof segmentationRuleLabels]}
|
||||
displayedValue={String(fieldValueGetters[field]!())}
|
||||
/>
|
||||
))}
|
||||
<FieldInfo
|
||||
label={t(($) => $['stepTwo.indexMode'], { ns: 'datasetCreation' })}
|
||||
displayedValue={indexModeLabel}
|
||||
valueIcon={<img className="size-4" src={indexMethodIconSrc} alt="" />}
|
||||
/>
|
||||
<FieldInfo
|
||||
label={t(($) => $['form.retrievalSetting.title'], { ns: 'datasetSettings' })}
|
||||
displayedValue={retrievalLabel}
|
||||
valueIcon={<img className="size-4" src={retrievalIconSrc} alt="" />}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default RuleDetail
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
import type { FC } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import UpgradeBtn from '@/app/components/billing/upgrade-btn'
|
||||
|
||||
const UpgradeBanner: FC = () => {
|
||||
const { t } = useTranslation(['billing'])
|
||||
|
||||
return (
|
||||
<div className="flex h-14 items-center rounded-xl border-[0.5px] border-black/5 bg-white p-3 shadow-md">
|
||||
<div className="flex h-8 w-8 shrink-0 items-center justify-center rounded-lg bg-[#FFF6ED]">
|
||||
<span
|
||||
aria-hidden
|
||||
className="i-custom-vender-solid-general-zap-fast h-4 w-4 text-[#FB6514]"
|
||||
/>
|
||||
</div>
|
||||
<div className="mx-3 grow text-[13px] font-medium text-gray-700">
|
||||
{t(($) => $['plansCommon.documentProcessingPriorityUpgrade'], { ns: 'billing' })}
|
||||
</div>
|
||||
<UpgradeBtn loc="knowledge-speed-up" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default UpgradeBanner
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
import type { DocumentStatusResponse } from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import { useEffect, useRef, useState } from 'react'
|
||||
import { fetchIndexingStatusBatch } from '@/service/datasets'
|
||||
|
||||
const POLLING_INTERVAL = 2500
|
||||
const COMPLETED_STATUSES = ['completed', 'error', 'paused'] as const
|
||||
const EMBEDDING_STATUSES = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const
|
||||
|
||||
type IndexingStatusPollingParams = {
|
||||
datasetId: string
|
||||
batchId: string
|
||||
}
|
||||
|
||||
type IndexingStatusPollingResult = {
|
||||
statusList: DocumentStatusResponse[]
|
||||
isEmbedding: boolean
|
||||
isEmbeddingCompleted: boolean
|
||||
}
|
||||
|
||||
const isStatusCompleted = (status: string): boolean =>
|
||||
COMPLETED_STATUSES.includes(status as (typeof COMPLETED_STATUSES)[number])
|
||||
|
||||
const isAllCompleted = (statusList: DocumentStatusResponse[]): boolean =>
|
||||
statusList.every((item) => isStatusCompleted(item.indexing_status))
|
||||
|
||||
/**
|
||||
* Custom hook for polling indexing status with automatic stop on completion.
|
||||
* Handles the polling lifecycle and provides derived states for UI rendering.
|
||||
*/
|
||||
export const useIndexingStatusPolling = ({
|
||||
datasetId,
|
||||
batchId,
|
||||
}: IndexingStatusPollingParams): IndexingStatusPollingResult => {
|
||||
const [statusList, setStatusList] = useState<DocumentStatusResponse[]>([])
|
||||
const isStopPollingRef = useRef(false)
|
||||
|
||||
useEffect(() => {
|
||||
// Reset polling state on mount
|
||||
isStopPollingRef.current = false
|
||||
let timeoutId: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const fetchStatus = async (): Promise<DocumentStatusResponse[]> => {
|
||||
const response = await fetchIndexingStatusBatch({ datasetId, batchId })
|
||||
setStatusList(response.data)
|
||||
return response.data
|
||||
}
|
||||
|
||||
const poll = async (): Promise<void> => {
|
||||
if (isStopPollingRef.current) return
|
||||
|
||||
try {
|
||||
const data = await fetchStatus()
|
||||
if (isAllCompleted(data)) {
|
||||
isStopPollingRef.current = true
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
// Continue polling on error
|
||||
}
|
||||
|
||||
if (!isStopPollingRef.current) {
|
||||
timeoutId = setTimeout(() => {
|
||||
poll()
|
||||
}, POLLING_INTERVAL)
|
||||
}
|
||||
}
|
||||
|
||||
poll()
|
||||
|
||||
return () => {
|
||||
isStopPollingRef.current = true
|
||||
if (timeoutId) clearTimeout(timeoutId)
|
||||
}
|
||||
}, [datasetId, batchId])
|
||||
|
||||
const isEmbedding = statusList.some((item) =>
|
||||
EMBEDDING_STATUSES.includes(item?.indexing_status as (typeof EMBEDDING_STATUSES)[number]),
|
||||
)
|
||||
|
||||
const isEmbeddingCompleted = statusList.length > 0 && isAllCompleted(statusList)
|
||||
|
||||
return {
|
||||
statusList,
|
||||
isEmbedding,
|
||||
isEmbeddingCompleted,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type {
|
||||
DocumentResponse,
|
||||
DocumentStatusResponse,
|
||||
} from '@dify/contracts/api/console/datasets/types.gen'
|
||||
import { toCreationDataSourceType } from '@/app/components/datasets/dataset-fields'
|
||||
import { getNotionPageIcon } from '@/app/components/datasets/documents/document-source'
|
||||
|
||||
const EMBEDDING_STATUSES = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const
|
||||
|
||||
/**
|
||||
* Check if a status indicates the source is being embedded
|
||||
*/
|
||||
export const isSourceEmbedding = (detail: DocumentStatusResponse): boolean =>
|
||||
EMBEDDING_STATUSES.includes(detail.indexing_status as (typeof EMBEDDING_STATUSES)[number])
|
||||
|
||||
/**
|
||||
* Calculate the progress percentage for a document
|
||||
*/
|
||||
export const getSourcePercent = (detail: DocumentStatusResponse): number => {
|
||||
const completedCount = detail.completed_segments || 0
|
||||
const totalCount = detail.total_segments || 0
|
||||
|
||||
if (totalCount === 0) return 0
|
||||
|
||||
const percent = Math.round((completedCount * 100) / totalCount)
|
||||
return Math.min(percent, 100)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file extension from filename, defaults to 'txt'
|
||||
*/
|
||||
export const getFileType = (name?: string): string => name?.split('.').pop() || 'txt'
|
||||
|
||||
/**
|
||||
* Document lookup utilities - provides document info by ID from a list
|
||||
*/
|
||||
export const createDocumentLookup = (documents: DocumentResponse[]) => {
|
||||
const documentMap = new Map(documents.map((doc) => [doc.id, doc]))
|
||||
|
||||
return {
|
||||
getDocument: (id: string) => documentMap.get(id),
|
||||
|
||||
getName: (id: string) => documentMap.get(id)?.name,
|
||||
|
||||
getSourceType: (id: string) => toCreationDataSourceType(documentMap.get(id)?.data_source_type),
|
||||
|
||||
getNotionIcon: (id: string) => getNotionPageIcon(documentMap.get(id)?.data_source_info, false),
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user