chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1,25 @@
import { render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import Header from '../header'
it('links to datasource documentation and opens that plugin configuration', async () => {
const user = userEvent.setup()
const onClickConfiguration = vi.fn()
render(
<Header
docTitle="Datasource documentation"
docLink="https://docs.example.com/source"
pluginName="Drive"
onClickConfiguration={onClickConfiguration}
credentials={[]}
currentCredentialId=""
onCredentialChange={vi.fn()}
/>,
)
expect(screen.getByRole('link', { name: 'Datasource documentation' })).toHaveAttribute(
'href',
'https://docs.example.com/source',
)
await user.click(screen.getByRole('button', { name: /datasetPipeline.configurationTip/ }))
expect(onClickConfiguration).toHaveBeenCalledOnce()
})
@@ -0,0 +1,93 @@
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
import { render, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import CredentialSelector from '../index'
const credentials = [
{
id: 'credential-1',
name: 'First credential',
avatar_url: 'https://example.com/first.png',
type: 'oauth2',
is_default: false,
},
{
id: 'credential-2',
name: 'Second credential',
avatar_url: 'https://example.com/second.png',
type: 'oauth2',
is_default: false,
},
] as DatasourceCredentialResponse[]
describe('CredentialSelector', () => {
it('selects the first credential when the current id is invalid', async () => {
const onCredentialChange = vi.fn()
render(
<CredentialSelector
currentCredentialId="missing"
credentials={credentials}
onCredentialChange={onCredentialChange}
/>,
)
await waitFor(() => expect(onCredentialChange).toHaveBeenCalledWith('credential-1'))
})
it('selects the visible default credential before falling back to the first one', async () => {
const onCredentialChange = vi.fn()
const credentialsWithDefault = credentials.map((credential, index) => ({
...credential,
is_default: index === 1,
}))
render(
<CredentialSelector
currentCredentialId="missing"
credentials={credentialsWithDefault}
onCredentialChange={onCredentialChange}
/>,
)
await waitFor(() => expect(onCredentialChange).toHaveBeenCalledWith('credential-2'))
})
it('does not select a fallback for an empty credential list', () => {
const onCredentialChange = vi.fn()
render(
<CredentialSelector
currentCredentialId="missing"
credentials={[]}
onCredentialChange={onCredentialChange}
/>,
)
expect(onCredentialChange).not.toHaveBeenCalled()
})
it('selects another credential and reflects the controlled value after closing the popup', async () => {
const user = userEvent.setup()
const onCredentialChange = vi.fn()
const { rerender } = render(
<CredentialSelector
currentCredentialId="credential-1"
credentials={credentials}
onCredentialChange={onCredentialChange}
/>,
)
const trigger = screen.getByRole('button', { name: /First credential/ })
expect(trigger).toHaveAttribute('aria-expanded', 'false')
expect(screen.queryByRole('img')).not.toBeInTheDocument()
await user.click(trigger)
await user.click(await screen.findByText('Second credential'))
expect(onCredentialChange).toHaveBeenCalledExactlyOnceWith('credential-2')
expect(trigger).toHaveAttribute('aria-expanded', 'false')
rerender(
<CredentialSelector
currentCredentialId="credential-2"
credentials={credentials}
onCredentialChange={onCredentialChange}
/>,
)
expect(screen.getByRole('button', { name: /Second credential/ })).toBeInTheDocument()
})
})
@@ -0,0 +1,67 @@
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
import { cn } from '@langgenius/dify-ui/cn'
import { Popover, PopoverContent, PopoverTrigger } from '@langgenius/dify-ui/popover'
import { useBoolean } from 'ahooks'
import * as React from 'react'
import { useCallback, useEffect, useMemo } from 'react'
import List from './list'
import Trigger from './trigger'
export type CredentialSelectorProps = {
currentCredentialId: string
onCredentialChange: (credentialId: string) => void
credentials: Array<DatasourceCredentialResponse>
}
const CredentialSelector = ({
currentCredentialId,
onCredentialChange,
credentials,
}: CredentialSelectorProps) => {
const [open, { set, setFalse }] = useBoolean(false)
const currentCredential = useMemo(() => {
return credentials.find((cred) => cred.id === currentCredentialId)
}, [credentials, currentCredentialId])
useEffect(() => {
if (!currentCredential && credentials.length) {
const fallbackCredential = credentials.find((credential) => credential.is_default)
onCredentialChange((fallbackCredential ?? credentials[0]!).id)
}
}, [currentCredential, credentials, onCredentialChange])
const handleCredentialChange = useCallback(
(credentialId: string) => {
onCredentialChange(credentialId)
setFalse()
},
[onCredentialChange, setFalse],
)
return (
<Popover open={open} onOpenChange={set}>
<PopoverTrigger
nativeButton={false}
render={(props, state) => (
<div {...props} className={cn('grow overflow-hidden', props.className)}>
<Trigger currentCredential={currentCredential} isOpen={state.open} />
</div>
)}
/>
<PopoverContent
align="start"
sideOffset={4}
className="border-none bg-transparent shadow-none"
>
<List
currentCredentialId={currentCredentialId}
credentials={credentials}
onCredentialChange={handleCredentialChange}
/>
</PopoverContent>
</Popover>
)
}
export default React.memo(CredentialSelector)
@@ -0,0 +1,32 @@
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
import { RiCheckLine } from '@remixicon/react'
import * as React from 'react'
import { useCallback } from 'react'
import { CredentialIcon } from '@/app/components/datasets/common/credential-icon'
type ItemProps = {
credential: DatasourceCredentialResponse
isSelected: boolean
onCredentialChange: (credentialId: string) => void
}
const Item = ({ credential, isSelected, onCredentialChange }: ItemProps) => {
const { avatar_url, name } = credential
const handleCredentialChange = useCallback(() => {
onCredentialChange(credential.id)
}, [credential.id, onCredentialChange])
return (
<div
className="flex cursor-pointer items-center gap-x-2 rounded-lg p-2 hover:bg-state-base-hover"
onClick={handleCredentialChange}
>
<CredentialIcon avatarUrl={avatar_url ?? undefined} name={name} size={20} />
<span className="grow truncate system-sm-medium text-text-secondary">{name}</span>
{isSelected && <RiCheckLine className="size-4 shrink-0 text-text-accent" />}
</div>
)
}
export default React.memo(Item)
@@ -0,0 +1,29 @@
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
import * as React from 'react'
import Item from './item'
type ListProps = {
currentCredentialId: string
credentials: Array<DatasourceCredentialResponse>
onCredentialChange: (credentialId: string) => void
}
const List = ({ currentCredentialId, credentials, onCredentialChange }: ListProps) => {
return (
<div className="flex w-70 flex-col gap-y-1 rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur p-1 shadow-lg shadow-shadow-shadow-5 backdrop-blur-[5px]">
{credentials.map((credential) => {
const isSelected = credential.id === currentCredentialId
return (
<Item
key={credential.id}
credential={credential}
isSelected={isSelected}
onCredentialChange={onCredentialChange}
/>
)
})}
</div>
)
}
export default React.memo(List)
@@ -0,0 +1,31 @@
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
import { cn } from '@langgenius/dify-ui/cn'
import { RiArrowDownSLine } from '@remixicon/react'
import * as React from 'react'
import { CredentialIcon } from '@/app/components/datasets/common/credential-icon'
type TriggerProps = {
currentCredential: DatasourceCredentialResponse | undefined
isOpen: boolean
}
const Trigger = ({ currentCredential, isOpen }: TriggerProps) => {
const { avatar_url, name = '' } = currentCredential || {}
return (
<div
className={cn(
'flex cursor-pointer items-center gap-x-2 rounded-md p-1 pr-2',
isOpen ? 'bg-state-base-hover' : 'hover:bg-state-base-hover',
)}
>
<CredentialIcon avatarUrl={avatar_url ?? undefined} name={name} size={20} />
<div className="flex grow items-center gap-x-1 overflow-hidden">
<span className="grow truncate system-md-semibold text-text-secondary">{name}</span>
<RiArrowDownSLine className="size-4 shrink-0 text-text-secondary" />
</div>
</div>
)
}
export default React.memo(Trigger)
@@ -0,0 +1,55 @@
import type { CredentialSelectorProps } from './credential-selector'
import { IconButton } from '@langgenius/dify-ui/icon-button'
import { Separator } from '@langgenius/dify-ui/separator'
import { Tooltip, TooltipContent, TooltipTrigger } from '@langgenius/dify-ui/tooltip'
import * as React from 'react'
import { useTranslation } from 'react-i18next'
import CredentialSelector from './credential-selector'
type HeaderProps = {
docTitle: string
docLink: string
onClickConfiguration?: () => void
pluginName: string
} & CredentialSelectorProps
const Header = ({ docTitle, docLink, onClickConfiguration, pluginName, ...rest }: HeaderProps) => {
const { t } = useTranslation(['datasetPipeline'])
const configurationTip = t(($) => $.configurationTip, { ns: 'datasetPipeline', pluginName })
return (
<div className="flex items-center justify-between gap-x-2">
<div className="flex items-center gap-x-1 overflow-hidden">
<CredentialSelector {...rest} />
<Separator decorative orientation="vertical" className="mx-1 h-3.5" />
<Tooltip>
<TooltipTrigger
render={
<IconButton
variant="ghost"
size="md"
className="shrink-0"
aria-label={configurationTip}
onClick={onClickConfiguration}
>
<span aria-hidden className="i-ri-equalizer-2-line size-4" />
</IconButton>
}
/>
<TooltipContent>{configurationTip}</TooltipContent>
</Tooltip>
</div>
<a
className="flex shrink-0 items-center gap-x-1 system-xs-medium text-text-accent"
href={docLink}
target="_blank"
rel="noopener noreferrer"
>
<span aria-hidden className="i-ri-book-open-line size-3.5 shrink-0" />
<span title={docTitle}>{docTitle}</span>
</a>
</div>
)
}
export default React.memo(Header)
@@ -0,0 +1,117 @@
import type { FileResponse } from '@dify/contracts/api/console/files/types.gen'
import { screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { toast } from '@/app/notifications'
import { AppToaster } from '@/app/notifications/app-toaster'
import { consoleQuery } from '@/service/console'
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
import { createFileUploadConfigFixture } from '@/test/fixtures/file-upload-config'
import { createDataSourceStore } from '../../store'
import { DataSourceContext } from '../../store/provider'
import LocalFile from '../index'
const upload = vi.hoisted(() => vi.fn<typeof import('@/service/base').upload>())
const supportedFormats = vi.hoisted(() => vi.fn(async () => ({ allowed_extensions: ['pdf'] })))
vi.mock('@/service/base', async (importOriginal) => ({
...(await importOriginal<typeof import('@/service/base')>()),
upload,
}))
vi.mock('@/service/console', async (importOriginal) => {
const actual = await importOriginal<typeof import('@/service/console')>()
const { withConsoleResponses } = await import('@/test/console/service-responses')
return {
...actual,
consoleQuery: withConsoleResponses(actual.consoleQuery, {
'files.supportType.get': supportedFormats,
}),
}
})
const setup = ({
allowedExtensions = ['pdf'],
supportBatchUpload = true,
existing = false,
} = {}) => {
const user = userEvent.setup({ applyAccept: false })
const store = createDataSourceStore()
const file = Object.assign(new File(['document'], 'Guide.pdf', { type: 'application/pdf' }), {
id: 'file-1',
})
if (existing) store.getState().setLocalFileList([{ fileID: 'local-1', file, progress: 100 }])
const queryClient = createConsoleQueryClient()
queryClient.setQueryData(
consoleQuery.files.upload.get.queryKey(),
createFileUploadConfigFixture(),
)
renderWithConsoleQuery(
<DataSourceContext value={store}>
<LocalFile allowedExtensions={allowedExtensions} supportBatchUpload={supportBatchUpload} />
<AppToaster timeout={0} />
</DataSourceContext>,
{ queryClient, systemFeatures: { deployment_edition: 'CLOUD' } },
)
return { user, store, queryClient, file }
}
beforeEach(() => {
upload.mockReset().mockResolvedValue({
id: 'csv-file',
name: 'data.csv',
size: 3,
extension: 'csv',
mime_type: 'text/csv',
} satisfies FileResponse)
})
afterEach(() => toast.dismiss())
it.each([true, false])(
'previews a stored file and applies batch-upload availability (%s)',
async (supportBatchUpload) => {
const { user, store, file } = setup({ supportBatchUpload, existing: true })
await user.click(screen.getByText('Guide.pdf'))
expect(store.getState().currentLocalFile).toBe(file)
const browse = screen.queryByRole('button', { name: 'datasetCreation.stepOne.uploader.browse' })
if (supportBatchUpload) expect(browse).toBeInTheDocument()
else expect(browse).not.toBeInTheDocument()
},
)
it.each([
{ allowedExtensions: ['csv'], accept: '.csv', canUpload: true },
{ allowedExtensions: [], accept: '', canUpload: false },
])(
'preserves the pipeline format override $allowedExtensions after supported formats load',
async ({ allowedExtensions, accept, canUpload }) => {
const { user, store, queryClient } = setup({ allowedExtensions })
const input = screen.getByLabelText('datasetCreation.stepOne.uploader.browse')
expect(input).toHaveAttribute('accept', accept)
await waitFor(() =>
expect(queryClient.getQueryData(consoleQuery.files.supportType.get.queryKey())).toEqual({
allowed_extensions: ['pdf'],
}),
)
expect(supportedFormats).toHaveBeenCalledOnce()
expect(input).toHaveAttribute('accept', accept)
await user.upload(input, new File(['row'], 'data.csv', { type: 'text/csv' }))
if (canUpload) {
await waitFor(() =>
expect(store.getState().localFileList[0]).toMatchObject({
file: { id: 'csv-file', name: 'data.csv' },
progress: 100,
}),
)
await user.click(screen.getByText('data.csv'))
expect(store.getState().currentLocalFile?.id).toBe('csv-file')
expect(upload).toHaveBeenCalledOnce()
} else {
expect(
await screen.findByText('datasetCreation.stepOne.uploader.validation.typeError'),
).toBeVisible()
expect(
screen.queryByRole('button', { name: 'datasetCreation.stepOne.uploader.browse' }),
).not.toBeInTheDocument()
expect(store.getState().localFileList).toEqual([])
expect(upload).not.toHaveBeenCalled()
}
},
)
@@ -0,0 +1,19 @@
import type { FileItem } from '@/models/datasets'
import { render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import FileListItem from '../file-list-item'
it('previews the uploaded file selected by name', async () => {
const user = userEvent.setup()
const onPreview = vi.fn()
const fileItem: FileItem = {
fileID: 'file-1',
progress: 100,
file: Object.assign(new File(['content'], 'guide.pdf', { type: 'application/pdf' }), {
id: 'uploaded-1',
}),
}
render(<FileListItem fileItem={fileItem} onPreview={onPreview} onRemove={vi.fn()} />)
await user.click(screen.getByText('guide.pdf'))
expect(onPreview).toHaveBeenCalledWith(fileItem.file)
})
@@ -0,0 +1,72 @@
import type { RefObject } from 'react'
import type { UploadDropzoneProps } from '../upload-dropzone'
import { screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { describe, expect, it, vi } from 'vite-plus/test'
import { renderWithConsoleQuery as render } from '@/test/console/query-data'
import UploadDropzone from '../upload-dropzone'
const createMockRef = <T,>(): RefObject<T | null> => ({ current: null })
describe('UploadDropzone', () => {
const defaultProps: UploadDropzoneProps = {
dropRef: createMockRef<HTMLDivElement>(),
dragRef: createMockRef<HTMLDivElement>(),
fileUploaderRef: createMockRef<HTMLInputElement>(),
dragging: false,
supportBatchUpload: true,
supportTypesShowNames: 'PDF, DOCX, TXT',
fileUploadConfig: {
file_size_limit: 15,
batch_count_limit: 5,
file_upload_limit: 10,
},
acceptTypes: ['.pdf', '.docx', '.txt'],
allowedExtensions: ['pdf', 'docx', 'txt'],
onSelectFile: vi.fn(),
onFileChange: vi.fn(),
}
it.each(['COMMUNITY', 'CLOUD'] as const)(
'shows the applicable upload limits for %s',
(edition) => {
render(<UploadDropzone {...defaultProps} />, {
systemFeatures: { deployment_edition: edition },
})
const tip = screen.getByText(/datasetCreation.stepOne.uploader.tip/)
expect(tip).toHaveTextContent('"size":15')
expect(tip).toHaveTextContent('"batchCount":5')
expect(tip).toHaveTextContent('"supportTypes":"PDF, DOCX, TXT"')
if (edition === 'CLOUD') expect(tip).toHaveTextContent('"totalCount":10')
else expect(tip).not.toHaveTextContent('"totalCount"')
},
)
it('disables browsing when no file types are allowed', () => {
render(<UploadDropzone {...defaultProps} allowedExtensions={[]} />)
expect(
screen.queryByRole('button', { name: 'datasetCreation.stepOne.uploader.browse' }),
).not.toBeInTheDocument()
})
it('forwards browsing and selected files to the upload owner', async () => {
const user = userEvent.setup()
const onFiles = vi.fn()
render(
<UploadDropzone
{...defaultProps}
onFileChange={(event) => onFiles(Array.from(event.target.files ?? []))}
/>,
)
await user.click(
screen.getByRole('button', { name: 'datasetCreation.stepOne.uploader.browse' }),
)
expect(defaultProps.onSelectFile).toHaveBeenCalledOnce()
const files = [
new File(['a'], 'a.pdf', { type: 'application/pdf' }),
new File(['b'], 'b.txt', { type: 'text/plain' }),
]
await user.upload(document.getElementById('fileUploader') as HTMLInputElement, files)
expect(onFiles).toHaveBeenCalledWith(files)
})
})
@@ -0,0 +1,86 @@
import type { CustomFile as File, FileItem } from '@/models/datasets'
import { cn } from '@langgenius/dify-ui/cn'
import { RiDeleteBinLine, RiErrorWarningFill } from '@remixicon/react'
import dynamic from 'next/dynamic'
import { useMemo } from 'react'
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
import { getFileType } from '@/app/components/datasets/common/image-uploader/utils'
import useTheme from '@/hooks/use-theme'
import { Theme } from '@/types/app'
import { formatFileSize } from '@/utils/format'
import { PROGRESS_ERROR } from '../constants'
const SimplePieChart = dynamic(() => import('@/app/components/base/simple-pie-chart'), {
ssr: false,
})
type FileListItemProps = {
fileItem: FileItem
onPreview: (file: File) => void
onRemove: (fileID: string) => void
}
const FileListItem = ({ fileItem, onPreview, onRemove }: FileListItemProps) => {
const { theme } = useTheme()
const chartColor = useMemo(() => (theme === Theme.dark ? '#5289ff' : '#296dff'), [theme])
const isUploading = fileItem.progress >= 0 && fileItem.progress < 100
const isError = fileItem.progress === PROGRESS_ERROR
const handleClick = () => {
onPreview(fileItem.file)
}
const handleRemove = (e: React.MouseEvent) => {
e.stopPropagation()
onRemove(fileItem.fileID)
}
return (
<div
onClick={handleClick}
className={cn(
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
isError && 'border-state-destructive-border bg-state-destructive-hover',
)}
>
<div className="flex w-12 shrink-0 items-center justify-center">
<DocumentFileIcon
size="lg"
className="shrink-0"
name={fileItem.file.name}
extension={getFileType(fileItem.file)}
/>
</div>
<div className="flex shrink grow flex-col gap-0.5">
<div className="flex w-full">
<div className="w-0 grow truncate text-xs text-text-secondary">{fileItem.file.name}</div>
</div>
<div className="w-full truncate text-2xs/3 text-text-tertiary">
<span className="uppercase">{getFileType(fileItem.file)}</span>
<span className="px-1 text-text-quaternary">·</span>
<span>{formatFileSize(fileItem.file.size)}</span>
</div>
</div>
<div className="flex w-16 shrink-0 items-center justify-end gap-1 pr-3">
{isUploading && (
<SimplePieChart
percentage={fileItem.progress}
stroke={chartColor}
fill={chartColor}
animationDuration={0}
/>
)}
{isError && <RiErrorWarningFill className="size-4 text-text-destructive" />}
<span
className="flex size-6 cursor-pointer items-center justify-center"
onClick={handleRemove}
>
<RiDeleteBinLine className="size-4 text-text-tertiary" />
</span>
</div>
</div>
)
}
export default FileListItem
@@ -0,0 +1,108 @@
import type { ChangeEvent, RefObject } from 'react'
import { cn } from '@langgenius/dify-ui/cn'
import { useAtomValue } from 'jotai'
import { useId } from 'react'
import { useTranslation } from 'react-i18next'
import { deploymentEditionAtom } from '@/features/system-features/state'
type FileUploadConfig = {
file_size_limit: number
batch_count_limit: number
file_upload_limit: number
}
export type UploadDropzoneProps = {
dropRef: RefObject<HTMLDivElement | null>
dragRef: RefObject<HTMLDivElement | null>
fileUploaderRef: RefObject<HTMLInputElement | null>
dragging: boolean
supportBatchUpload: boolean
supportTypesShowNames: string
fileUploadConfig: FileUploadConfig
acceptTypes: string[]
onSelectFile: () => void
onFileChange: (e: ChangeEvent<HTMLInputElement>) => void
allowedExtensions: string[]
}
const UploadDropzone = ({
dropRef,
dragRef,
fileUploaderRef,
dragging,
supportBatchUpload,
supportTypesShowNames,
fileUploadConfig,
acceptTypes,
onSelectFile,
onFileChange,
allowedExtensions,
}: UploadDropzoneProps) => {
const { t } = useTranslation(['datasetCreation'])
const browseLabelId = useId()
const canBrowse = allowedExtensions.length > 0
const deploymentEdition = useAtomValue(deploymentEditionAtom)
return (
<>
<input
ref={fileUploaderRef}
id="fileUploader"
className="hidden"
type="file"
aria-labelledby={canBrowse ? browseLabelId : undefined}
aria-label={
canBrowse ? undefined : t(($) => $['stepOne.uploader.browse'], { ns: 'datasetCreation' })
}
multiple={supportBatchUpload}
accept={acceptTypes.join(',')}
onChange={onFileChange}
/>
<div
ref={dropRef}
className={cn(
'relative box-border flex min-h-20 flex-col items-center justify-center gap-1 rounded-xl border border-dashed border-components-dropzone-border bg-components-dropzone-bg px-4 py-3 text-xs/4 text-text-tertiary',
dragging && 'border-components-dropzone-border-accent bg-components-dropzone-bg-accent',
)}
>
<div className="flex min-h-5 items-center justify-center text-sm/4 text-text-secondary">
<span className="mr-2 i-ri-upload-cloud-2-line size-5" />
<span>
{supportBatchUpload
? t(($) => $['stepOne.uploader.button'], { ns: 'datasetCreation' })
: t(($) => $['stepOne.uploader.buttonSingleFile'], { ns: 'datasetCreation' })}
{canBrowse && (
<button
id={browseLabelId}
type="button"
className="font-inherit ml-1 inline cursor-pointer appearance-none border-0 bg-transparent p-0 text-text-accent"
onClick={onSelectFile}
>
{t(($) => $['stepOne.uploader.browse'], { ns: 'datasetCreation' })}
</button>
)}
</span>
</div>
<div>
{deploymentEdition === 'CLOUD'
? t(($) => $['stepOne.uploader.tipWithTotalLimit'], {
ns: 'datasetCreation',
size: fileUploadConfig.file_size_limit,
supportTypes: supportTypesShowNames,
batchCount: fileUploadConfig.batch_count_limit,
totalCount: fileUploadConfig.file_upload_limit,
})
: t(($) => $['stepOne.uploader.tip'], {
ns: 'datasetCreation',
size: fileUploadConfig.file_size_limit,
supportTypes: supportTypesShowNames,
batchCount: fileUploadConfig.batch_count_limit,
})}
</div>
{dragging && <div ref={dragRef} className="absolute top-0 left-0 size-full" />}
</div>
</>
)
}
export default UploadDropzone
@@ -0,0 +1 @@
export const PROGRESS_ERROR = -2
@@ -0,0 +1,104 @@
import { act, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { consoleQuery } from '@/service/console'
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
import { createFileUploadConfigFixture } from '@/test/fixtures/file-upload-config'
import { createDataSourceStore } from '../../../store'
import { DataSourceContext } from '../../../store/provider'
import { useLocalFileUpload } from '../use-local-file-upload'
const upload = vi.fn()
vi.mock('@/service/base', () => ({ upload: (...args: unknown[]) => upload(...args) }))
vi.mock('@/app/notifications', () => ({ toast: { error: vi.fn() } }))
function Uploader() {
const { fileUploaderRef, fileChangeHandle, localFileList, removeFile } = useLocalFileUpload({
allowedExtensions: ['pdf'],
})
return (
<>
<input
aria-label="Upload documents"
type="file"
ref={fileUploaderRef}
onChange={fileChangeHandle}
/>
{localFileList.map((item) => (
<button key={item.fileID} onClick={() => removeFile(item.fileID)}>
Remove {item.file.name}
</button>
))}
</>
)
}
function setup() {
const store = createDataSourceStore()
const existing = {
fileID: 'existing',
file: Object.assign(new File(['existing'], 'Existing.pdf'), { id: 'server-existing' }),
progress: 100,
}
store.getState().setLocalFileList([existing])
const queryClient = createConsoleQueryClient()
queryClient.setQueryData(
consoleQuery.files.upload.get.queryOptions().queryKey,
createFileUploadConfigFixture(),
)
queryClient.setQueryData(consoleQuery.files.supportType.get.queryOptions().queryKey, {
allowed_extensions: ['pdf'],
})
renderWithConsoleQuery(
<DataSourceContext value={store}>
<Uploader />
</DataSourceContext>,
{ queryClient, systemFeatures: { deployment_edition: 'CLOUD' } },
)
return { store, existing }
}
// Generic validation, drop handling and limits belong to useFileUpload. This
// adapter owns merging upload transitions into the pipeline's data-source store.
it('updates only the uploaded file and preserves other files through progress, success and removal', async () => {
const user = userEvent.setup()
let resolve!: (value: { id: string; name: string }) => void
upload.mockImplementationOnce(
() =>
new Promise((res) => {
resolve = res
}),
)
const { store, existing } = setup()
await user.upload(
screen.getByLabelText('Upload documents'),
new File(['new'], 'New.pdf', { type: 'application/pdf' }),
)
expect(store.getState().localFileList).toHaveLength(2)
const request = upload.mock.lastCall![0]
act(() => request.onprogress({ lengthComputable: true, loaded: 53, total: 100 }))
expect(store.getState().localFileList[1]?.progress).toBe(53)
expect(store.getState().localFileList[0]).toEqual(existing)
await act(async () => resolve({ id: 'server-new', name: 'New.pdf' }))
await waitFor(() =>
expect(store.getState().localFileList[1]).toEqual({
fileID: expect.any(String),
file: { id: 'server-new', name: 'New.pdf' },
progress: 100,
}),
)
await user.click(screen.getByRole('button', { name: 'Remove New.pdf' }))
expect(store.getState().localFileList).toEqual([existing])
})
it('marks a failed upload without losing existing documents', async () => {
const user = userEvent.setup()
upload.mockRejectedValueOnce(new Error('offline'))
const { store, existing } = setup()
await user.upload(
screen.getByLabelText('Upload documents'),
new File(['new'], 'New.pdf', { type: 'application/pdf' }),
)
await waitFor(() => expect(store.getState().localFileList[1]?.progress).toBe(-2))
expect(store.getState().localFileList[0]).toEqual(existing)
expect(store.getState().localFileList[1]?.file.name).toBe('New.pdf')
})
@@ -0,0 +1,111 @@
import type { CustomFile as File, FileItem } from '@/models/datasets'
import { produce } from 'immer'
import { useCallback } from 'react'
import { useFileUpload } from '@/app/components/datasets/create/file-uploader/hooks/use-file-upload'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../../store'
type UseLocalFileUploadOptions = {
allowedExtensions: string[]
supportBatchUpload?: boolean
}
/**
* Hook for handling local file uploads in the create-from-pipeline flow.
* This is a thin wrapper around the generic useFileUpload hook that provides
* Zustand store integration for state management.
*/
export const useLocalFileUpload = ({
allowedExtensions,
supportBatchUpload = true,
}: UseLocalFileUploadOptions) => {
const localFileList = useDataSourceStoreWithSelector((state) => state.localFileList)
const dataSourceStore = useDataSourceStore()
const prepareFileList = useCallback(
(files: FileItem[]) => {
const { setLocalFileList } = dataSourceStore.getState()
setLocalFileList(files)
},
[dataSourceStore],
)
const onFileUpdate = useCallback(
(fileItem: FileItem, progress: number, list: FileItem[]) => {
const { setLocalFileList } = dataSourceStore.getState()
const newList = produce(list, (draft) => {
const targetIndex = draft.findIndex((file) => file.fileID === fileItem.fileID)
if (targetIndex !== -1) {
draft[targetIndex] = {
...draft[targetIndex],
...fileItem,
progress,
}
}
})
setLocalFileList(newList)
},
[dataSourceStore],
)
const onFileListUpdate = useCallback(
(files: FileItem[]) => {
const { setLocalFileList } = dataSourceStore.getState()
setLocalFileList(files)
},
[dataSourceStore],
)
const onPreview = useCallback(
(file: File) => {
const { setCurrentLocalFile } = dataSourceStore.getState()
setCurrentLocalFile(file)
},
[dataSourceStore],
)
const {
dropRef,
dragRef,
fileUploaderRef,
dragging,
fileUploadConfig,
acceptTypes,
supportTypesShowNames,
hideUpload,
selectHandle,
fileChangeHandle,
removeFile,
handlePreview,
} = useFileUpload({
fileList: localFileList,
prepareFileList,
onFileUpdate,
onFileListUpdate,
onPreview,
supportBatchUpload,
allowedExtensions,
})
return {
// Refs
dropRef,
dragRef,
fileUploaderRef,
// State
dragging,
localFileList,
// Config
fileUploadConfig,
acceptTypes,
supportTypesShowNames,
hideUpload,
// Handlers
selectHandle,
fileChangeHandle,
removeFile,
handlePreview,
}
}
@@ -0,0 +1,61 @@
'use client'
import FileListItem from './components/file-list-item'
import UploadDropzone from './components/upload-dropzone'
import { useLocalFileUpload } from './hooks/use-local-file-upload'
type LocalFileProps = {
allowedExtensions: string[]
supportBatchUpload?: boolean
}
const LocalFile = ({ allowedExtensions, supportBatchUpload = true }: LocalFileProps) => {
const {
dropRef,
dragRef,
fileUploaderRef,
dragging,
localFileList,
fileUploadConfig,
acceptTypes,
supportTypesShowNames,
hideUpload,
selectHandle,
fileChangeHandle,
removeFile,
handlePreview,
} = useLocalFileUpload({ allowedExtensions, supportBatchUpload })
return (
<div className="flex flex-col">
{!hideUpload && (
<UploadDropzone
dropRef={dropRef}
dragRef={dragRef}
fileUploaderRef={fileUploaderRef}
dragging={dragging}
supportBatchUpload={supportBatchUpload}
supportTypesShowNames={supportTypesShowNames}
fileUploadConfig={fileUploadConfig}
acceptTypes={acceptTypes}
onSelectFile={selectHandle}
onFileChange={fileChangeHandle}
allowedExtensions={allowedExtensions}
/>
)}
{localFileList.length > 0 && (
<div className="mt-1 flex flex-col gap-y-1">
{localFileList.map((fileItem, index) => (
<FileListItem
key={`${fileItem.fileID}-${index}`}
fileItem={fileItem}
onPreview={handlePreview}
onRemove={removeFile}
/>
))}
</div>
)}
</div>
)
}
export default LocalFile
@@ -0,0 +1,228 @@
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { DataSourceNotionWorkspace } from '@/models/common'
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
import { act, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
import { BlockEnum } from '@/app/components/workflow/types'
import { DatasourceType } from '@/models/pipeline'
import { consoleQuery } from '@/service/console'
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
import { createNuqsTestRenderer } from '@/test/nuqs-testing'
import { createDataSourceStore } from '../../store'
import { DataSourceContext } from '../../store/provider'
import OnlineDocuments from '../index'
const { ssePost, toastError } = vi.hoisted(() => ({ ssePost: vi.fn(), toastError: vi.fn() }))
vi.mock('@tanstack/react-virtual')
vi.mock('@/service/base', async (importOriginal) => ({
...(await importOriginal<typeof import('@/service/base')>()),
ssePost,
}))
vi.mock('@/app/notifications', () => ({ toast: { error: toastError } }))
vi.mock('@/context/dataset-detail', () => ({
useDatasetDetailContextWithSelector: (
select: (state: { dataset: { pipeline_id: string } }) => unknown,
) => select({ dataset: { pipeline_id: 'pipeline-123' } }),
}))
const onUrlUpdate = vi.fn()
const render = createNuqsTestRenderer(renderWithConsoleQuery, () => ({ onUrlUpdate }))
const nodeData: DataSourceNodeType = {
type: BlockEnum.DataSource,
title: 'Notion',
desc: '',
provider_type: DatasourceType.onlineDocument,
plugin_id: 'plugin-123',
provider_name: 'notion',
datasource_name: 'notion',
datasource_label: 'Notion',
datasource_parameters: { workspace: { type: VarKindType.constant, value: 'workspace-1' } },
datasource_configurations: {},
}
const documents: DataSourceNotionWorkspace[] = [
{
workspace_name: 'Workspace',
workspace_id: 'workspace-1',
workspace_icon: null,
pages: [
{
page_id: 'report',
page_name: 'Report',
parent_id: 'root',
type: 'page',
page_icon: null,
is_bound: false,
},
{
page_id: 'notes',
page_name: 'Notes',
parent_id: 'root',
type: 'page',
page_icon: null,
is_bound: false,
},
],
},
]
const credentials: DatasourceCredentialResponse[] = [
{
id: 'credential-1',
name: 'First credential',
avatar_url: '',
type: 'oauth2',
is_default: false,
credential: {},
},
{
id: 'credential-2',
name: 'Second credential',
avatar_url: '',
type: 'oauth2',
is_default: false,
credential: {},
},
]
function setup({ cached = true, credential = 'credential-1', single = false, draft = false } = {}) {
const store = createDataSourceStore()
store.setState({ documentsData: cached ? documents : undefined, currentCredentialId: credential })
const queryClient = createConsoleQueryClient()
queryClient.setQueryData(
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
input: { params: { provider_id: 'plugin-123/notion' } },
}).queryKey,
{ result: credential ? credentials : [] },
)
const onCredentialChange = vi.fn((id: string) => store.getState().setCurrentCredentialId(id))
render(
<DataSourceContext value={store}>
<OnlineDocuments
nodeId="node-1"
nodeData={nodeData}
onCredentialChange={onCredentialChange}
supportBatchUpload={!single}
isInPipeline={draft}
/>
</DataSourceContext>,
{ queryClient },
)
return { store, onCredentialChange }
}
function callbacks(): {
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => void
onDataSourceNodeError: (data: DataSourceNodeErrorResponse) => void
} {
return ssePost.mock.lastCall![2]
}
beforeEach(() => vi.clearAllMocks())
it.each([true, false])(
'loads documents through the draft=%s route using resolved node parameters',
async (draft) => {
const { store } = setup({ cached: false, draft })
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
expect(ssePost).toHaveBeenCalledWith(
`/rag/pipelines/pipeline-123/workflows/${draft ? 'draft' : 'published'}/datasource/nodes/node-1/run`,
{
body: {
inputs: { workspace: 'workspace-1' },
credential_id: 'credential-1',
datasource_type: DatasourceType.onlineDocument,
},
},
expect.any(Object),
)
act(() =>
callbacks().onDataSourceNodeCompleted({
event: 'datasource_completed',
time_consuming: 1,
data: documents,
}),
)
expect(await screen.findByRole('checkbox', { name: 'Report' })).toBeInTheDocument()
expect(store.getState().documentsData).toEqual(documents)
expect(screen.queryByTestId('notion-page-preview-report') !== null).toBe(!draft)
},
)
it('waits for a credential before requesting documents', () => {
setup({ cached: false, credential: '' })
expect(ssePost).not.toHaveBeenCalled()
})
it('selects pages with their workspace and removes only the deselected page', async () => {
const user = userEvent.setup()
const { store } = setup()
await user.click(screen.getByRole('checkbox', { name: 'Report' }))
await user.click(screen.getByRole('checkbox', { name: 'Notes' }))
expect(store.getState().selectedPagesId).toEqual(new Set(['report', 'notes']))
expect(store.getState().onlineDocuments).toEqual(
documents[0]!.pages.map((page) => ({ ...page, workspace_id: 'workspace-1' })),
)
await user.click(screen.getByRole('checkbox', { name: 'Report' }))
expect(store.getState().onlineDocuments).toEqual([
{ ...documents[0]!.pages[1], workspace_id: 'workspace-1' },
])
expect(screen.getByRole('checkbox', { name: 'Notes' })).toBeChecked()
})
it('replaces the selected page in single-upload mode', async () => {
const user = userEvent.setup()
const { store } = setup({ single: true })
await user.click(screen.getByRole('radio', { name: 'Report' }))
await user.click(screen.getByRole('radio', { name: 'Notes' }))
expect(store.getState().selectedPagesId).toEqual(new Set(['notes']))
expect(store.getState().onlineDocuments).toEqual([
{ ...documents[0]!.pages[1], workspace_id: 'workspace-1' },
])
expect(screen.getByRole('radio', { name: 'Report' })).not.toBeChecked()
})
it('filters documents without losing selection and previews the chosen page', async () => {
const user = userEvent.setup()
const { store } = setup()
await user.click(screen.getByRole('checkbox', { name: 'Report' }))
await user.type(
screen.getByPlaceholderText('common.dataSource.notion.selector.searchPages'),
'Notes',
)
await waitFor(() =>
expect(screen.queryByRole('checkbox', { name: 'Report' })).not.toBeInTheDocument(),
)
await user.click(screen.getByTestId('notion-page-preview-notes'))
expect(store.getState().currentDocument).toEqual({
...documents[0]!.pages[1],
workspace_id: 'workspace-1',
})
expect(store.getState().selectedPagesId).toEqual(new Set(['report']))
await user.click(screen.getByRole('button', { name: 'common.operation.clear' }))
expect(await screen.findByRole('checkbox', { name: 'Report' })).toBeChecked()
})
it('requests documents again with the credential selected in the header', async () => {
const user = userEvent.setup()
const { onCredentialChange } = setup()
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
await user.click(screen.getByRole('button', { name: /First credential/ }))
await user.click(await screen.findByText('Second credential'))
expect(onCredentialChange).toHaveBeenCalledExactlyOnceWith('credential-2')
await waitFor(() => expect(ssePost).toHaveBeenCalledTimes(2))
expect(ssePost.mock.lastCall![1].body.credential_id).toBe('credential-2')
})
it('reports stream failures and opens data-source configuration', async () => {
const user = userEvent.setup()
setup({ cached: false })
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
act(() =>
callbacks().onDataSourceNodeError({
event: 'datasource_error',
error: 'Notion request failed',
}),
)
expect(toastError).toHaveBeenCalledExactlyOnceWith('Notion request failed')
await user.click(screen.getByRole('button', { name: /^datasetPipeline\.configurationTip/ }))
await waitFor(() =>
expect(onUrlUpdate.mock.lastCall![0].searchParams.get('settings')).toBe('data-source'),
)
})
@@ -0,0 +1,195 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
import { useQuery } from '@tanstack/react-query'
import { useQueryState } from 'nuqs'
import { useCallback, useEffect, useMemo } from 'react'
import { useShallow } from 'zustand/react/shallow'
import { LoadingPlaceholder } from '@/app/components/base/loading-placeholder'
import SearchInput from '@/app/components/base/notion-page-selector/search-input'
import {
settingsQueryParamName,
settingsQueryParser,
} from '@/app/components/header/account-setting/query-params'
import { toast } from '@/app/notifications'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useDocLink } from '@/context/i18n'
import { DatasourceType } from '@/models/pipeline'
import { ssePost } from '@/service/base'
import { consoleQuery } from '@/service/console'
import Header from '../base/header'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
import PageSelector from './page-selector'
import Title from './title'
type OnlineDocumentsProps = {
nodeId: string
nodeData: DataSourceNodeType
onCredentialChange: (credentialId: string) => void
isInPipeline?: boolean
supportBatchUpload?: boolean
}
const OnlineDocuments = ({
nodeId,
nodeData,
isInPipeline = false,
supportBatchUpload = true,
onCredentialChange,
}: OnlineDocumentsProps) => {
const docLink = useDocLink()
const pipelineId = useDatasetDetailContextWithSelector((s) => s.dataset?.pipeline_id)
const [, setSettingsDestination] = useQueryState(settingsQueryParamName, settingsQueryParser)
const { documentsData, searchValue, selectedPagesId, currentCredentialId } =
useDataSourceStoreWithSelector(
useShallow((state) => ({
documentsData: state.documentsData,
searchValue: state.searchValue,
selectedPagesId: state.selectedPagesId,
currentCredentialId: state.currentCredentialId,
})),
)
const { data: dataSourceAuth } = useQuery(
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
input: { params: { provider_id: `${nodeData.plugin_id}/${nodeData.provider_name}` } },
}),
)
const dataSourceStore = useDataSourceStore()
const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
const pagesMap = (documentsData || []).reduce(
(prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
next.pages.forEach((page) => {
prev[page.page_id] = {
...page,
workspace_id: next.workspace_id,
}
})
return prev
},
{},
)
return pagesMap
}, [documentsData])
const datasourceNodeRunURL = !isInPipeline
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
const getOnlineDocuments = useCallback(async () => {
const { currentCredentialId } = dataSourceStore.getState()
// Convert datasource_parameters to inputs format for the API
const inputs = Object.entries(nodeData.datasource_parameters || {}).reduce(
(acc, [key, value]) => {
acc[key] =
typeof value === 'object' && value !== null && 'value' in value ? value.value : value
return acc
},
{} as Record<string, any>,
)
ssePost(
datasourceNodeRunURL,
{
body: {
inputs,
credential_id: currentCredentialId,
datasource_type: DatasourceType.onlineDocument,
},
},
{
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
const { setDocumentsData } = dataSourceStore.getState()
setDocumentsData(documentsData.data as DataSourceNotionWorkspace[])
},
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
toast.error(error.error)
},
},
)
}, [dataSourceStore, datasourceNodeRunURL, nodeData.datasource_parameters])
useEffect(() => {
if (!currentCredentialId) return
getOnlineDocuments()
}, [currentCredentialId, getOnlineDocuments])
const handleSearchValueChange = useCallback(
(value: string) => {
const { setSearchValue } = dataSourceStore.getState()
setSearchValue(value)
},
[dataSourceStore],
)
const handleSelectPages = useCallback(
(newSelectedPagesId: Set<string>) => {
const { setSelectedPagesId, setOnlineDocuments } = dataSourceStore.getState()
const selectedPages = Array.from(newSelectedPagesId).map(
(pageId) => PagesMapAndSelectedPagesId[pageId]!,
)
setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
setOnlineDocuments(selectedPages)
},
[dataSourceStore, PagesMapAndSelectedPagesId],
)
const handlePreviewPage = useCallback(
(previewPageId: string) => {
const { setCurrentDocument } = dataSourceStore.getState()
setCurrentDocument(PagesMapAndSelectedPagesId[previewPageId])
},
[PagesMapAndSelectedPagesId, dataSourceStore],
)
const handleSetting = useCallback(() => {
setSettingsDestination('data-source')
}, [setSettingsDestination])
return (
<div className="flex flex-col gap-y-2">
<Header
docTitle="Docs"
docLink={docLink('/use-dify/knowledge/knowledge-pipeline/authorize-data-source')}
onClickConfiguration={handleSetting}
pluginName={nodeData.datasource_label}
currentCredentialId={currentCredentialId}
onCredentialChange={onCredentialChange}
credentials={dataSourceAuth?.result || []}
/>
<div className="rounded-xl border border-components-panel-border bg-background-default-subtle">
<div className="flex items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-1 pl-3">
<div className="flex grow items-center">
<Title name={nodeData.datasource_label} />
</div>
<SearchInput value={searchValue} onChange={handleSearchValueChange} />
</div>
<div className="overflow-hidden rounded-b-xl">
{documentsData?.length ? (
<PageSelector
key={`${currentCredentialId}:${supportBatchUpload ? 'multiple' : 'single'}`}
checkedIds={selectedPagesId}
disabledValue={new Set()}
searchValue={searchValue}
list={documentsData[0]!.pages || []}
pagesMap={PagesMapAndSelectedPagesId}
onSelect={handleSelectPages}
canPreview={!isInPipeline}
onPreview={handlePreviewPage}
isMultipleChoice={supportBatchUpload}
/>
) : (
<div className="flex h-74 items-center justify-center">
<LoadingPlaceholder className="h-full" />
</div>
)}
</div>
</div>
</div>
)
}
export default OnlineDocuments
@@ -0,0 +1,126 @@
import type { ComponentProps } from 'react'
import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
import { render, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { useState } from 'react'
import PageSelector from '../index'
vi.mock('@tanstack/react-virtual')
const page = (id: string, name: string, parent = 'root'): DataSourceNotionPage => ({
page_id: id,
page_name: name,
parent_id: parent,
page_icon: null,
is_bound: false,
type: 'page',
})
const pages = [
page('parent', 'Parent'),
page('child', 'Child', 'parent'),
page('grandchild', 'Grandchild', 'child'),
page('other', 'Other'),
]
const mapPages = (list: DataSourceNotionPage[]): DataSourceNotionPageMap =>
Object.fromEntries(list.map((item) => [item.page_id, { ...item, workspace_id: 'workspace' }]))
const defaults = {
list: pages,
pagesMap: mapPages(pages),
disabledValue: new Set<string>(),
searchValue: '',
}
function Harness(props: Partial<ComponentProps<typeof PageSelector>>) {
const [selected, setSelected] = useState(props.checkedIds ?? new Set<string>())
return (
<PageSelector
{...defaults}
{...props}
checkedIds={selected}
onSelect={(ids) => {
setSelected(ids)
props.onSelect?.(ids)
}}
/>
)
}
it('expands nested pages and collapses descendants when their parent is collapsed', async () => {
const user = userEvent.setup()
render(<Harness />)
expect(screen.queryByRole('checkbox', { name: 'Child' })).not.toBeInTheDocument()
await user.click(screen.getByTestId('notion-page-toggle-parent'))
await user.click(screen.getByTestId('notion-page-toggle-child'))
expect(screen.getByRole('checkbox', { name: 'Grandchild' })).toBeInTheDocument()
await user.click(screen.getByTestId('notion-page-toggle-parent'))
expect(screen.queryByRole('checkbox', { name: 'Child' })).not.toBeInTheDocument()
expect(screen.queryByRole('checkbox', { name: 'Grandchild' })).not.toBeInTheDocument()
await user.click(screen.getByTestId('notion-page-toggle-parent'))
expect(screen.getByRole('checkbox', { name: 'Child' })).toBeInTheDocument()
expect(screen.queryByRole('checkbox', { name: 'Grandchild' })).not.toBeInTheDocument()
})
it('selects and deselects a complete subtree without changing an independent selection', async () => {
const user = userEvent.setup()
const onSelect = vi.fn()
render(<Harness checkedIds={new Set(['other'])} onSelect={onSelect} />)
await user.click(screen.getByRole('checkbox', { name: 'Parent' }))
expect(onSelect).toHaveBeenLastCalledWith(new Set(['other', 'parent', 'child', 'grandchild']))
await user.click(screen.getByRole('checkbox', { name: 'Parent' }))
expect(onSelect).toHaveBeenLastCalledWith(new Set(['other']))
expect(screen.getByRole('checkbox', { name: 'Other' })).toBeChecked()
})
it('selects only the matched page in search mode and shows its ancestors', async () => {
const user = userEvent.setup()
const onSelect = vi.fn()
render(<Harness searchValue="Child" onSelect={onSelect} />)
expect(screen.queryByRole('checkbox', { name: 'Parent' })).not.toBeInTheDocument()
expect(screen.getByTitle('Parent / Child')).toBeInTheDocument()
await user.click(screen.getByRole('checkbox', { name: 'Child' }))
expect(onSelect).toHaveBeenCalledExactlyOnceWith(new Set(['child']))
})
it('selects one page without cascading in single-choice mode', async () => {
const user = userEvent.setup()
const onSelect = vi.fn()
render(<Harness isMultipleChoice={false} onSelect={onSelect} />)
await user.click(screen.getByRole('radio', { name: 'Parent' }))
expect(onSelect).toHaveBeenLastCalledWith(new Set(['parent']))
await user.click(screen.getByRole('radio', { name: 'Other' }))
expect(onSelect).toHaveBeenLastCalledWith(new Set(['other']))
expect(screen.getByRole('radio', { name: 'Parent' })).not.toBeChecked()
})
it.each([true, false])('blocks a disabled page in multiple-choice=%s mode', async (multiple) => {
const user = userEvent.setup()
const onSelect = vi.fn()
render(
<Harness isMultipleChoice={multiple} disabledValue={new Set(['other'])} onSelect={onSelect} />,
)
await user.click(screen.getByRole(multiple ? 'checkbox' : 'radio', { name: 'Other' }))
expect(onSelect).not.toHaveBeenCalled()
await user.click(screen.getByRole(multiple ? 'checkbox' : 'radio', { name: 'Parent' }))
expect(onSelect).toHaveBeenCalledOnce()
})
it('previews the requested page without selecting it and hides preview when disabled', async () => {
const user = userEvent.setup()
const onSelect = vi.fn()
const onPreview = vi.fn()
const { rerender } = render(<Harness onSelect={onSelect} onPreview={onPreview} />)
await user.click(screen.getByTestId('notion-page-preview-other'))
expect(onPreview).toHaveBeenCalledExactlyOnceWith('other')
expect(onSelect).not.toHaveBeenCalled()
rerender(<Harness canPreview={false} onSelect={onSelect} onPreview={onPreview} />)
expect(screen.queryByTestId('notion-page-preview-other')).not.toBeInTheDocument()
})
it('updates visible results when documents or the search change', async () => {
const { rerender } = render(<Harness list={[]} pagesMap={{}} />)
expect(screen.getByText('common.dataSource.notion.selector.noSearchResult')).toBeInTheDocument()
const list = [page('orphan', 'Orphan', 'missing-parent')]
rerender(<Harness list={list} pagesMap={mapPages(list)} />)
expect(screen.getByRole('checkbox', { name: 'Orphan' })).toBeInTheDocument()
rerender(<Harness list={list} pagesMap={mapPages(list)} searchValue="missing" />)
await waitFor(() => expect(screen.queryByRole('checkbox')).not.toBeInTheDocument())
expect(screen.getByText('common.dataSource.notion.selector.noSearchResult')).toBeInTheDocument()
})
@@ -0,0 +1,74 @@
import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
import { useTranslation } from 'react-i18next'
import { usePageSelectorModel } from '@/app/components/base/notion-page-selector/page-selector/use-page-selector-model'
import VirtualPageList from '@/app/components/base/notion-page-selector/page-selector/virtual-page-list'
type PageSelectorProps = {
checkedIds: Set<string>
disabledValue: Set<string>
searchValue: string
pagesMap: DataSourceNotionPageMap
list: DataSourceNotionPage[]
onSelect: (selectedPagesId: Set<string>) => void
canPreview?: boolean
onPreview?: (selectedPageId: string) => void
isMultipleChoice?: boolean
currentCredentialId?: string
}
const PageSelector = ({
checkedIds,
disabledValue,
searchValue,
pagesMap,
list,
onSelect,
canPreview = true,
onPreview,
isMultipleChoice = true,
currentCredentialId: _currentCredentialId,
}: PageSelectorProps) => {
const { t } = useTranslation(['common'])
const selectionMode = isMultipleChoice ? 'multiple' : 'single'
const {
currentPreviewPageId,
effectiveSearchValue,
rows,
handlePreview,
handleSelect,
handleToggle,
} = usePageSelectorModel({
checkedIds,
list,
onPreview,
onSelect,
pagesMap,
searchValue,
selectionMode,
})
if (!rows.length) {
return (
<div className="flex h-74 items-center justify-center text-[13px] text-text-tertiary">
{t(($) => $['dataSource.notion.selector.noSearchResult'], { ns: 'common' })}
</div>
)
}
return (
<VirtualPageList
checkedIds={checkedIds}
disabledValue={disabledValue}
onPreview={handlePreview}
onSelect={handleSelect}
onToggle={handleToggle}
previewPageId={currentPreviewPageId}
rows={rows}
searchValue={effectiveSearchValue}
selectionMode={selectionMode}
showPreview={canPreview}
/>
)
}
export default PageSelector
@@ -0,0 +1,18 @@
import * as React from 'react'
import { useTranslation } from 'react-i18next'
type TitleProps = {
name: string
}
const Title = ({ name }: TitleProps) => {
const { t } = useTranslation(['datasetPipeline'])
return (
<div className="px-1.25 py-1 system-sm-medium text-text-secondary">
{t(($) => $['onlineDocument.pageSelectorTitle'], { ns: 'datasetPipeline', name })}
</div>
)
}
export default React.memo(Title)
@@ -0,0 +1,165 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { OnlineDriveFile } from '@/models/pipeline'
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
import { act, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { BlockEnum } from '@/app/components/workflow/types'
import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline'
import { consoleQuery } from '@/service/console'
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
import { createNuqsTestRenderer } from '@/test/nuqs-testing'
import { createDataSourceStore } from '../../store'
import { DataSourceContext } from '../../store/provider'
import OnlineDrive from '../index'
const { ssePost, toastError } = vi.hoisted(() => ({ ssePost: vi.fn(), toastError: vi.fn() }))
vi.mock('@/service/base', async (importOriginal) => ({
...(await importOriginal<typeof import('@/service/base')>()),
ssePost,
}))
vi.mock('@/app/notifications', () => ({ toast: { error: toastError } }))
vi.mock('@/context/dataset-detail', () => ({
useDatasetDetailContextWithSelector: (
select: (state: { dataset: { pipeline_id: string } }) => unknown,
) => select({ dataset: { pipeline_id: 'pipeline-123' } }),
}))
const onUrlUpdate = vi.fn()
const render = createNuqsTestRenderer(renderWithConsoleQuery, () => ({ onUrlUpdate }))
const nodeData: DataSourceNodeType = {
type: BlockEnum.DataSource,
title: 'Drive',
desc: '',
provider_type: DatasourceType.onlineDrive,
plugin_id: 'plugin-123',
provider_name: 'drive',
datasource_name: 'drive',
datasource_label: 'Drive',
datasource_parameters: {},
datasource_configurations: {},
}
const files: OnlineDriveFile[] = [
{ id: 'file-1', name: 'Report.pdf', type: OnlineDriveFileType.file, size: 1024 },
{ id: 'file-2', name: 'Other.txt', type: OnlineDriveFileType.file, size: 500 },
{ id: 'folder-1', name: 'Archive', type: OnlineDriveFileType.folder },
]
function setup({ cached = true, credential = 'credential-1', single = false, draft = false } = {}) {
const store = createDataSourceStore()
store.setState({
localFileList: [],
onlineDriveFileList: cached ? files : [],
currentCredentialId: credential,
breadcrumbs: ['Root'],
prefix: ['root'],
bucket: 'bucket-a',
nextPageParameters: { cursor: 'next' },
})
const queryClient = createConsoleQueryClient()
queryClient.setQueryData(
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
input: { params: { provider_id: 'plugin-123/drive' } },
}).queryKey,
{ result: [] },
)
render(
<DataSourceContext value={store}>
<OnlineDrive
nodeId="node-1"
nodeData={nodeData}
onCredentialChange={vi.fn()}
supportBatchUpload={!single}
isInPipeline={draft}
/>
</DataSourceContext>,
{ queryClient },
)
return store
}
function callbacks(): {
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => void
onDataSourceNodeError: (data: DataSourceNodeErrorResponse) => void
} {
return ssePost.mock.lastCall![2]
}
beforeEach(() => vi.clearAllMocks())
it.each([true, false])(
'loads the %s draft route and stores streamed files and pagination',
async (draft) => {
const store = setup({ cached: false, draft })
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
expect(ssePost).toHaveBeenCalledWith(
`/rag/pipelines/pipeline-123/workflows/${draft ? 'draft' : 'published'}/datasource/nodes/node-1/run`,
{
body: {
inputs: {
prefix: 'root',
bucket: 'bucket-a',
next_page_parameters: { cursor: 'next' },
max_keys: 30,
},
datasource_type: DatasourceType.onlineDrive,
credential_id: 'credential-1',
},
},
expect.any(Object),
)
act(() =>
callbacks().onDataSourceNodeCompleted({
event: 'datasource_completed',
time_consuming: 1,
data: [
{
bucket: 'bucket-a',
files: [{ id: 'new-file', name: 'New.pdf', size: 1024, type: 'file' }],
is_truncated: true,
next_page_parameters: { cursor: 'next-page' },
},
],
}),
)
expect(await screen.findByText('New.pdf')).toBeInTheDocument()
expect(store.getState().isTruncated.current).toBe(true)
expect(store.getState().currentNextPageParametersRef.current).toEqual({ cursor: 'next-page' })
expect(store.getState().hasBucket).toBe(true)
},
)
it('does not request files without a credential', () => {
setup({ cached: false, credential: '' })
expect(ssePost).not.toHaveBeenCalled()
})
it('uses cached files, selects them independently and clears selection when opening a folder', async () => {
const user = userEvent.setup()
const store = setup()
expect(ssePost).not.toHaveBeenCalled()
await user.click(screen.getByRole('checkbox', { name: 'Report.pdf' }))
expect(store.getState().selectedFileIds).toEqual(['file-1'])
expect(screen.getByRole('checkbox', { name: 'Other.txt' })).not.toBeChecked()
await user.click(screen.getByText('Archive'))
expect(store.getState().selectedFileIds).toEqual([])
expect(store.getState().breadcrumbs).toEqual(['Root', 'Archive'])
expect(store.getState().prefix).toEqual(['root', 'folder-1'])
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
expect(ssePost.mock.lastCall![1].body.inputs.prefix).toBe('folder-1')
})
it('keeps a single selected file when a second file is clicked in single-upload mode', async () => {
const user = userEvent.setup()
const store = setup({ single: true })
await user.click(screen.getByText('Report.pdf'))
await user.click(screen.getByText('Other.txt'))
expect(store.getState().selectedFileIds).toEqual(['file-1'])
})
it('reports a failed stream and opens datasource configuration', async () => {
const user = userEvent.setup()
setup({ cached: false })
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
act(() => callbacks().onDataSourceNodeError({ event: 'datasource_error', error: 'Drive failed' }))
expect(toastError).toHaveBeenCalledWith('Drive failed')
await user.click(screen.getByRole('button', { name: /^datasetPipeline\.configurationTip/ }))
await waitFor(() =>
expect(onUrlUpdate.mock.lastCall![0].searchParams.get('settings')).toBe('data-source'),
)
})
@@ -0,0 +1,95 @@
import type { OnlineDriveData } from '@/types/pipeline'
import { describe, expect, it } from 'vite-plus/test'
import { OnlineDriveFileType } from '@/models/pipeline'
import { convertOnlineDriveData, isBucketListInitiation } from '../utils'
describe('online-drive utils', () => {
describe('isBucketListInitiation', () => {
it('should return true when data has buckets and no prefix/bucket set', () => {
const data: OnlineDriveData[] = [
{ bucket: 'bucket-1', files: [], is_truncated: false, next_page_parameters: {} },
{ bucket: 'bucket-2', files: [], is_truncated: false, next_page_parameters: {} },
]
expect(isBucketListInitiation(data, [], '')).toBe(true)
})
it('should return false when bucket is already set', () => {
const data: OnlineDriveData[] = [
{ bucket: 'bucket-1', files: [], is_truncated: false, next_page_parameters: {} },
]
expect(isBucketListInitiation(data, [], 'bucket-1')).toBe(false)
})
it('should return false when prefix is set', () => {
const data: OnlineDriveData[] = [
{ bucket: 'bucket-1', files: [], is_truncated: false, next_page_parameters: {} },
]
expect(isBucketListInitiation(data, ['folder/'], '')).toBe(false)
})
it('should return false when single bucket has files', () => {
const data: OnlineDriveData[] = [
{
bucket: 'bucket-1',
files: [{ id: 'f1', name: 'test.txt', size: 100, type: 'file' as const }],
is_truncated: false,
next_page_parameters: {},
},
]
expect(isBucketListInitiation(data, [], '')).toBe(false)
})
})
describe('convertOnlineDriveData', () => {
it('should return empty result for empty data', () => {
const result = convertOnlineDriveData([], [], '')
expect(result.fileList).toEqual([])
expect(result.isTruncated).toBe(false)
expect(result.hasBucket).toBe(false)
})
it('should convert bucket list initiation to bucket items', () => {
const data: OnlineDriveData[] = [
{ bucket: 'bucket-1', files: [], is_truncated: false, next_page_parameters: {} },
{ bucket: 'bucket-2', files: [], is_truncated: false, next_page_parameters: {} },
]
const result = convertOnlineDriveData(data, [], '')
expect(result.fileList).toHaveLength(2)
expect(result.fileList[0]).toEqual({
id: 'bucket-1',
name: 'bucket-1',
type: OnlineDriveFileType.bucket,
})
expect(result.hasBucket).toBe(true)
})
it('should convert files when not bucket list', () => {
const data: OnlineDriveData[] = [
{
bucket: 'bucket-1',
files: [
{ id: 'f1', name: 'test.txt', size: 100, type: 'file' as const },
{ id: 'f2', name: 'folder', size: 0, type: 'folder' as const },
],
is_truncated: true,
next_page_parameters: { token: 'next' },
},
]
const result = convertOnlineDriveData(data, [], 'bucket-1')
expect(result.fileList).toHaveLength(2)
expect(result.fileList[0]!.type).toBe(OnlineDriveFileType.file)
expect(result.fileList[0]!.size).toBe(100)
expect(result.fileList[1]!.type).toBe(OnlineDriveFileType.folder)
expect(result.fileList[1]!.size).toBeUndefined()
expect(result.isTruncated).toBe(true)
expect(result.nextPageParameters).toEqual({ token: 'next' })
expect(result.hasBucket).toBe(true)
})
})
})
@@ -0,0 +1,149 @@
import { act, fireEvent, render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import * as React from 'react'
import FileList from '../index'
// Mock store - context provider requires mocking
const mockStoreState = {
setNextPageParameters: vi.fn(),
currentNextPageParametersRef: { current: {} },
isTruncated: { current: false },
hasBucket: false,
setOnlineDriveFileList: vi.fn(),
setSelectedFileIds: vi.fn(),
setBreadcrumbs: vi.fn(),
setPrefix: vi.fn(),
setBucket: vi.fn(),
}
const mockGetState = vi.fn(() => mockStoreState)
const mockDataSourceStore = { getState: mockGetState }
vi.mock('../../../store', () => ({
useDataSourceStore: () => mockDataSourceStore,
useDataSourceStoreWithSelector: (selector: (s: typeof mockStoreState) => unknown) =>
selector(mockStoreState),
}))
type FileListProps = React.ComponentProps<typeof FileList>
const createDefaultProps = (overrides?: Partial<FileListProps>): FileListProps => ({
fileList: [],
selectedFileIds: [],
breadcrumbs: [],
keywords: '',
bucket: '',
isInPipeline: false,
resetKeywords: vi.fn(),
updateKeywords: vi.fn(),
searchResultsLength: 0,
handleSelectFile: vi.fn(),
handleOpenFolder: vi.fn(),
isLoading: false,
supportBatchUpload: true,
...overrides,
})
const resetMockStoreState = () => {
mockStoreState.setNextPageParameters = vi.fn()
mockStoreState.currentNextPageParametersRef = { current: {} }
mockStoreState.isTruncated = { current: false }
mockStoreState.hasBucket = false
mockStoreState.setOnlineDriveFileList = vi.fn()
mockStoreState.setSelectedFileIds = vi.fn()
mockStoreState.setBreadcrumbs = vi.fn()
mockStoreState.setPrefix = vi.fn()
mockStoreState.setBucket = vi.fn()
}
describe('FileList', () => {
beforeEach(() => {
vi.clearAllMocks()
resetMockStoreState()
})
afterEach(() => {
vi.useRealTimers()
})
describe('State Management', () => {
describe('debounced keywords update', () => {
it('should delay the search until typing has stopped', () => {
vi.useFakeTimers({ shouldAdvanceTime: true })
const mockUpdateKeywords = vi.fn()
const props = createDefaultProps({ updateKeywords: mockUpdateKeywords })
render(<FileList {...props} />)
const input = screen.getByPlaceholderText(
'datasetPipeline.onlineDrive.breadcrumbs.searchPlaceholder',
)
fireEvent.change(input, { target: { value: 'debounced-value' } })
expect(mockUpdateKeywords).not.toHaveBeenCalled()
act(() => vi.advanceTimersByTime(499))
expect(mockUpdateKeywords).not.toHaveBeenCalled()
act(() => vi.advanceTimersByTime(1))
expect(mockUpdateKeywords).toHaveBeenCalledExactlyOnceWith('debounced-value')
})
})
})
// Event Handlers Tests
describe('Event Handlers', () => {
describe('handleInputChange', () => {
it('should search only for the latest input after rapid changes', () => {
vi.useFakeTimers({ shouldAdvanceTime: true })
const mockUpdateKeywords = vi.fn()
const props = createDefaultProps({ updateKeywords: mockUpdateKeywords })
render(<FileList {...props} />)
const input = screen.getByPlaceholderText(
'datasetPipeline.onlineDrive.breadcrumbs.searchPlaceholder',
)
fireEvent.change(input, { target: { value: 'a' } })
fireEvent.change(input, { target: { value: 'ab' } })
fireEvent.change(input, { target: { value: 'abc' } })
act(() => vi.advanceTimersByTime(500))
expect(mockUpdateKeywords).toHaveBeenCalledExactlyOnceWith('abc')
expect(input).toHaveValue('abc')
})
})
describe('handleResetKeywords', () => {
it('should cancel the pending search before clearing the query', async () => {
vi.useFakeTimers({ shouldAdvanceTime: true })
const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime })
const mockResetKeywords = vi.fn()
const props = createDefaultProps({ resetKeywords: mockResetKeywords })
render(<FileList {...props} />)
const input = screen.getByRole('searchbox', {
name: 'datasetPipeline.onlineDrive.breadcrumbs.searchPlaceholder',
})
await user.type(input, 'report')
await user.click(screen.getByRole('button', { name: 'common.operation.clear' }))
act(() => vi.advanceTimersByTime(500))
expect(props.updateKeywords).not.toHaveBeenCalled()
expect(mockResetKeywords).toHaveBeenCalledOnce()
expect(input).toHaveValue('')
})
})
})
// File Type Variations
// Search Results Display
// Callback Stability
})
it('clears an unmatched drive search from the empty-result action', async () => {
const user = userEvent.setup()
const props = createDefaultProps({ keywords: 'missing' })
render(<FileList {...props} />)
await user.click(
screen.getByRole('button', { name: 'datasetPipeline.onlineDrive.resetKeywords' }),
)
expect(props.resetKeywords).toHaveBeenCalledOnce()
})
@@ -0,0 +1,156 @@
import type { ComponentProps } from 'react'
import { render, screen, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { createDataSourceStore } from '../../../../../store'
import { DataSourceContext } from '../../../../../store/provider'
import Breadcrumbs from '../index'
let store = createDataSourceStore()
function renderPath(overrides: Partial<ComponentProps<typeof Breadcrumbs>> = {}) {
const props = {
breadcrumbs: ['Documents', 'Reports'],
keywords: '',
bucket: '',
searchResultsLength: 0,
isInPipeline: false,
...overrides,
}
store.setState({
breadcrumbs: props.breadcrumbs,
prefix: props.breadcrumbs.map((_, index) => `prefix-${index}`),
bucket: props.bucket,
selectedFileIds: ['selected-file'],
onlineDriveFileList: [{ id: 'selected-file', name: 'report.txt', type: 'file', size: 10 }],
})
return render(<Breadcrumbs {...props} />, {
wrapper: ({ children }) => <DataSourceContext value={store}>{children}</DataSourceContext>,
})
}
const allFiles = 'datasetPipeline.onlineDrive.breadcrumbs.allFiles'
const allBuckets = 'datasetPipeline.onlineDrive.breadcrumbs.allBuckets'
beforeEach(() => {
store = createDataSourceStore()
})
it('exposes path items without counting decorative separators and marks the current directory', async () => {
const user = userEvent.setup()
renderPath()
const path = within(screen.getByRole('navigation', { name: allFiles }))
expect(path.getAllByRole('listitem')).toHaveLength(3)
const current = path.getByRole('button', { name: 'Reports' })
expect(current).toHaveAttribute('aria-current', 'location')
expect(current).toBeDisabled()
await user.click(current)
expect(store.getState().breadcrumbs).toEqual(['Documents', 'Reports'])
await user.click(path.getByRole('button', { name: 'Documents' }))
expect(store.getState().breadcrumbs).toEqual(['Documents'])
expect(store.getState().prefix).toEqual(['prefix-0'])
expect(store.getState().onlineDriveFileList).toEqual([])
expect(store.getState().selectedFileIds).toEqual([])
})
it('returns to the drive root and disables root navigation once already there', async () => {
const user = userEvent.setup()
const { rerender } = renderPath()
await user.click(screen.getByRole('button', { name: allFiles }))
expect(store.getState().breadcrumbs).toEqual([])
expect(store.getState().prefix).toEqual([])
expect(store.getState().onlineDriveFileList).toEqual([])
expect(store.getState().selectedFileIds).toEqual([])
rerender(
<Breadcrumbs
breadcrumbs={[]}
keywords=""
bucket=""
searchResultsLength={0}
isInPipeline={false}
/>,
)
expect(screen.getAllByRole('listitem')).toHaveLength(1)
expect(screen.getByRole('button', { name: allFiles })).toBeDisabled()
expect(screen.getByRole('button', { name: allFiles })).toHaveAttribute('aria-current', 'location')
})
it('distinguishes returning to the current bucket from leaving it for the bucket list', async () => {
const user = userEvent.setup()
store.setState({ hasBucket: true })
renderPath({ bucket: 'Storage' })
await user.click(screen.getByRole('button', { name: 'Storage' }))
expect(store.getState().breadcrumbs).toEqual([])
expect(store.getState().prefix).toEqual([])
expect(store.getState().bucket).toBe('Storage')
await user.click(screen.getByRole('button', { name: allBuckets }))
expect(store.getState().bucket).toEqual('')
expect(store.getState().onlineDriveFileList).toEqual([])
expect(store.getState().selectedFileIds).toEqual([])
})
it('keeps the bucket-list action available at the current bucket root', () => {
store.setState({ hasBucket: true })
renderPath({ bucket: 'Storage', breadcrumbs: [] })
expect(screen.getByRole('button', { name: allBuckets })).toBeEnabled()
expect(screen.getByRole('button', { name: 'Storage' })).toBeDisabled()
expect(screen.getByRole('button', { name: 'Storage' })).toHaveAttribute(
'aria-current',
'location',
)
})
it.each([
{ isInPipeline: false, bucket: '', visiblePrefixes: ['A', 'B'] },
{ isInPipeline: true, bucket: '', visiblePrefixes: ['A'] },
{ isInPipeline: false, bucket: 'Storage', visiblePrefixes: ['A'] },
{ isInPipeline: true, bucket: 'Storage', visiblePrefixes: [] },
])(
'keeps collapsed paths navigable with pipeline=$isInPipeline and bucket=$bucket',
async ({ isInPipeline, bucket, visiblePrefixes }) => {
const user = userEvent.setup()
store.setState({ hasBucket: !!bucket })
renderPath({ breadcrumbs: ['A', 'B', 'C', 'D'], isInPipeline, bucket })
for (const name of visiblePrefixes)
expect(screen.getByRole('button', { name })).toBeInTheDocument()
expect(screen.queryByRole('button', { name: 'C' })).not.toBeInTheDocument()
expect(screen.getByRole('button', { name: 'D' })).toHaveAttribute('aria-current', 'location')
const more = screen.getByRole('button', { name: 'common.operation.more' })
more.focus()
await user.keyboard('{Enter}')
await user.click(await screen.findByRole('menuitem', { name: 'C' }))
expect(store.getState().breadcrumbs).toEqual(['A', 'B', 'C'])
expect(store.getState().prefix).toEqual(['prefix-0', 'prefix-1', 'prefix-2'])
},
)
it('shows a bucket-list heading instead of an empty navigation landmark', () => {
store.setState({ hasBucket: true })
renderPath({ breadcrumbs: [] })
expect(screen.getByText(allBuckets)).toBeInTheDocument()
expect(screen.queryByRole('navigation')).not.toBeInTheDocument()
})
it.each([
{ breadcrumbs: [], bucket: 'Storage', folderName: 'Storage' },
{ breadcrumbs: ['Documents', 'Reports'], bucket: 'Storage', folderName: 'Reports' },
])(
'replaces the path with search results scoped to $folderName',
({ breadcrumbs, bucket, folderName }) => {
renderPath({ breadcrumbs, bucket, keywords: 'query', searchResultsLength: 5 })
expect(screen.getByText(/onlineDrive.breadcrumbs.searchResult/)).toHaveTextContent(folderName)
expect(screen.queryByRole('navigation')).not.toBeInTheDocument()
},
)
it('keeps the directory path available when a search has no results', () => {
renderPath({ keywords: 'query', searchResultsLength: 0 })
expect(screen.getByRole('navigation', { name: allFiles })).toBeInTheDocument()
expect(screen.getByRole('button', { name: 'Reports' })).toHaveAttribute(
'aria-current',
'location',
)
})
@@ -0,0 +1,60 @@
import { BreadcrumbItem, BreadcrumbSeparator } from '@langgenius/dify-ui/breadcrumb'
import { IconButton } from '@langgenius/dify-ui/icon-button'
import { Tooltip, TooltipContent, TooltipTrigger } from '@langgenius/dify-ui/tooltip'
import * as React from 'react'
import { useTranslation } from 'react-i18next'
import DirectoryItem from './item'
type BucketProps = {
bucketName: string
current: boolean
handleBackToBucketList: () => void
handleClickBucketName: () => void
}
const Bucket = ({
bucketName,
handleBackToBucketList,
handleClickBucketName,
current,
}: BucketProps) => {
const { t } = useTranslation(['datasetPipeline'])
const allBucketsLabel = t(($) => $['onlineDrive.breadcrumbs.allBuckets'], {
ns: 'datasetPipeline',
})
return (
<>
<BreadcrumbItem className="shrink-0">
<Tooltip>
<TooltipTrigger
render={
<IconButton
type="button"
variant="ghost"
size="md"
aria-label={allBucketsLabel}
onClick={handleBackToBucketList}
>
<span
aria-hidden
className="i-custom-public-knowledge-online-drive-buckets-gray h-4.75 w-4.5"
/>
</IconButton>
}
/>
<TooltipContent>{allBucketsLabel}</TooltipContent>
</Tooltip>
</BreadcrumbItem>
<BreadcrumbSeparator className="system-xs-regular text-divider-deep" />
<DirectoryItem
name={bucketName}
title={bucketName}
current={current}
onClick={handleClickBucketName}
/>
</>
)
}
export default React.memo(Bucket)
@@ -0,0 +1,19 @@
import { useTranslation } from 'react-i18next'
import BreadcrumbItem from './item'
type DriveProps = {
breadcrumbs: string[]
handleBackToRoot: () => void
}
export default function Drive({ breadcrumbs, handleBackToRoot }: DriveProps) {
const { t } = useTranslation(['datasetPipeline'])
return (
<BreadcrumbItem
name={t(($) => $['onlineDrive.breadcrumbs.allFiles'], { ns: 'datasetPipeline' })}
current={breadcrumbs.length === 0}
onClick={handleBackToRoot}
/>
)
}
@@ -0,0 +1,176 @@
import { Breadcrumb, BreadcrumbList } from '@langgenius/dify-ui/breadcrumb'
import * as React from 'react'
import { useCallback, useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../../../../store'
import Bucket from './bucket'
import Drive from './drive'
import BreadcrumbItem from './item'
import ActionsMenu from './menu'
type BreadcrumbsProps = {
breadcrumbs: string[]
keywords: string
bucket: string
searchResultsLength: number
isInPipeline: boolean
}
const Breadcrumbs = ({
breadcrumbs,
keywords,
bucket,
searchResultsLength,
isInPipeline,
}: BreadcrumbsProps) => {
const { t } = useTranslation(['datasetPipeline'])
const dataSourceStore = useDataSourceStore()
const hasBucket = useDataSourceStoreWithSelector((s) => s.hasBucket)
const showSearchResult = !!keywords && searchResultsLength > 0
const showBucketListTitle = breadcrumbs.length === 0 && hasBucket && bucket === ''
const displayBreadcrumbNum = useMemo(() => {
const num = isInPipeline ? 2 : 3
return bucket ? num - 1 : num
}, [isInPipeline, bucket])
const breadcrumbsConfig = useMemo(() => {
const prefixToDisplay = breadcrumbs.slice(0, displayBreadcrumbNum - 1)
const collapsedBreadcrumbs = breadcrumbs.slice(displayBreadcrumbNum - 1, breadcrumbs.length - 1)
return {
original: breadcrumbs,
needCollapsed: breadcrumbs.length > displayBreadcrumbNum,
prefixBreadcrumbs: prefixToDisplay,
collapsedBreadcrumbs,
lastBreadcrumb: breadcrumbs[breadcrumbs.length - 1],
}
}, [displayBreadcrumbNum, breadcrumbs])
const handleBackToBucketList = useCallback(() => {
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix, setBucket } =
dataSourceStore.getState()
setOnlineDriveFileList([])
setSelectedFileIds([])
setBucket('')
setBreadcrumbs([])
setPrefix([])
}, [dataSourceStore])
const handleClickBucketName = useCallback(() => {
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } =
dataSourceStore.getState()
setOnlineDriveFileList([])
setSelectedFileIds([])
setBreadcrumbs([])
setPrefix([])
}, [dataSourceStore])
const handleBackToRoot = useCallback(() => {
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } =
dataSourceStore.getState()
setOnlineDriveFileList([])
setSelectedFileIds([])
setBreadcrumbs([])
setPrefix([])
}, [dataSourceStore])
const handleClickBreadcrumb = useCallback(
(index: number) => {
const {
breadcrumbs,
prefix,
setOnlineDriveFileList,
setSelectedFileIds,
setBreadcrumbs,
setPrefix,
} = dataSourceStore.getState()
const newBreadcrumbs = breadcrumbs.slice(0, index + 1)
const newPrefix = prefix.slice(0, index + 1)
setOnlineDriveFileList([])
setSelectedFileIds([])
setBreadcrumbs(newBreadcrumbs)
setPrefix(newPrefix)
},
[dataSourceStore],
)
return (
<div className="flex min-w-0 grow items-center">
{showSearchResult && (
<div className="px-1.25 system-sm-medium">
{t(($) => $['onlineDrive.breadcrumbs.searchResult'], {
ns: 'datasetPipeline',
searchResultsLength,
folderName: breadcrumbs.at(-1) ?? bucket ?? '',
})}
</div>
)}
{!showSearchResult && showBucketListTitle && (
<div className="px-1.25 system-sm-medium">
{t(($) => $['onlineDrive.breadcrumbs.allBuckets'], { ns: 'datasetPipeline' })}
</div>
)}
{!showSearchResult && !showBucketListTitle && (
<Breadcrumb
aria-label={t(($) => $['onlineDrive.breadcrumbs.allFiles'], { ns: 'datasetPipeline' })}
className="w-full"
>
<BreadcrumbList className="gap-0.5">
{hasBucket && bucket && (
<Bucket
bucketName={bucket}
handleBackToBucketList={handleBackToBucketList}
handleClickBucketName={handleClickBucketName}
current={breadcrumbs.length === 0}
/>
)}
{!hasBucket && <Drive breadcrumbs={breadcrumbs} handleBackToRoot={handleBackToRoot} />}
{!breadcrumbsConfig.needCollapsed && (
<>
{breadcrumbsConfig.original.map((breadcrumb, index) => {
const isLast = index === breadcrumbsConfig.original.length - 1
return (
<BreadcrumbItem
key={breadcrumbs.slice(0, index + 1).join('/')}
onClick={() => handleClickBreadcrumb(index)}
name={breadcrumb}
title={breadcrumb}
current={isLast}
/>
)
})}
</>
)}
{breadcrumbsConfig.needCollapsed && (
<>
{breadcrumbsConfig.prefixBreadcrumbs.map((breadcrumb, index) => {
return (
<BreadcrumbItem
key={breadcrumbs.slice(0, index + 1).join('/')}
onClick={() => handleClickBreadcrumb(index)}
name={breadcrumb}
title={breadcrumb}
/>
)
})}
<ActionsMenu
startIndex={breadcrumbsConfig.prefixBreadcrumbs.length}
breadcrumbs={breadcrumbsConfig.collapsedBreadcrumbs}
onBreadcrumbClick={handleClickBreadcrumb}
/>
<BreadcrumbItem
onClick={() => handleClickBreadcrumb(breadcrumbs.length - 1)}
name={breadcrumbsConfig.lastBreadcrumb!}
title={breadcrumbsConfig.lastBreadcrumb!}
current
/>
</>
)}
</BreadcrumbList>
</Breadcrumb>
)}
</div>
)
}
export default React.memo(Breadcrumbs)
@@ -0,0 +1,39 @@
import { BreadcrumbSeparator, BreadcrumbItem as PathItem } from '@langgenius/dify-ui/breadcrumb'
import { cn } from '@langgenius/dify-ui/cn'
type BreadcrumbItemProps = {
name: string
onClick: () => void
current?: boolean
title?: string
}
export default function BreadcrumbItem({
name,
onClick,
current = false,
title,
}: BreadcrumbItemProps) {
return (
<>
<PathItem>
<button
type="button"
aria-current={current ? 'location' : undefined}
className={cn(
'truncate rounded-md px-1.25 py-1 focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:outline-hidden',
current
? 'system-sm-medium text-text-secondary'
: 'system-sm-regular text-text-tertiary hover:bg-state-base-hover',
)}
disabled={current}
onClick={onClick}
title={title}
>
{name}
</button>
</PathItem>
{!current && <BreadcrumbSeparator className="system-xs-regular text-divider-deep" />}
</>
)
}
@@ -0,0 +1,48 @@
import { BreadcrumbItem, BreadcrumbSeparator } from '@langgenius/dify-ui/breadcrumb'
import { IconButton } from '@langgenius/dify-ui/icon-button'
import { Menu, MenuContent, MenuItem, MenuTrigger } from '@langgenius/dify-ui/menu'
import * as React from 'react'
import { useTranslation } from 'react-i18next'
type ActionsMenuProps = {
startIndex: number
breadcrumbs: string[]
onBreadcrumbClick: (index: number) => void
}
const ActionsMenu = ({ startIndex, breadcrumbs, onBreadcrumbClick }: ActionsMenuProps) => {
const { t } = useTranslation(['common'])
return (
<>
<BreadcrumbItem className="shrink-0">
<Menu>
<MenuTrigger
render={
<IconButton
aria-label={t(($) => $['operation.more'], { ns: 'common' })}
className="data-popup-open:bg-state-base-hover"
>
<span aria-hidden className="i-ri-more-fill size-4" />
</IconButton>
}
/>
<MenuContent align="start" className="w-34">
{breadcrumbs.map((breadcrumb, index) => (
<MenuItem
key={breadcrumbs.slice(0, index + 1).join('/')}
className="px-3 py-1.5 system-md-regular"
onClick={() => onBreadcrumbClick(startIndex + index)}
>
{breadcrumb}
</MenuItem>
))}
</MenuContent>
</Menu>
</BreadcrumbItem>
<BreadcrumbSeparator className="system-xs-regular text-divider-deep" />
</>
)
}
export default React.memo(ActionsMenu)
@@ -0,0 +1,50 @@
import * as React from 'react'
import { useTranslation } from 'react-i18next'
import { SearchInput } from '@/app/components/base/search-input'
import Breadcrumbs from './breadcrumbs'
type HeaderProps = {
breadcrumbs: string[]
inputValue: string
keywords: string
bucket: string
searchResultsLength: number
onSearchValueChange: (value: string) => void
isInPipeline: boolean
}
const Header = ({
breadcrumbs,
inputValue,
keywords,
bucket,
isInPipeline,
searchResultsLength,
onSearchValueChange,
}: HeaderProps) => {
const { t } = useTranslation(['datasetPipeline'])
const searchLabel = t(($) => $['onlineDrive.breadcrumbs.searchPlaceholder'], {
ns: 'datasetPipeline',
})
return (
<div className="flex items-center gap-x-2 bg-components-panel-bg p-1 pl-3">
<Breadcrumbs
breadcrumbs={breadcrumbs}
keywords={keywords}
bucket={bucket}
searchResultsLength={searchResultsLength}
isInPipeline={isInPipeline}
/>
<SearchInput
className="h-8 w-50 shrink-0"
value={inputValue}
onValueChange={onSearchValueChange}
aria-label={searchLabel}
placeholder={searchLabel}
/>
</div>
)
}
export default React.memo(Header)
@@ -0,0 +1,88 @@
import type { OnlineDriveFile } from '@/models/pipeline'
import { useDebounceFn } from 'ahooks'
import { useState } from 'react'
import Header from './header'
import List from './list'
type FileListProps = {
fileList: OnlineDriveFile[]
selectedFileIds: string[]
breadcrumbs: string[]
keywords: string
bucket: string
isInPipeline: boolean
resetKeywords: () => void
updateKeywords: (keywords: string) => void
searchResultsLength: number
handleSelectFile: (file: OnlineDriveFile) => void
handleOpenFolder: (file: OnlineDriveFile) => void
isLoading: boolean
supportBatchUpload: boolean
}
const FileList = ({
fileList,
selectedFileIds,
breadcrumbs,
keywords,
bucket,
resetKeywords,
updateKeywords,
searchResultsLength,
handleSelectFile,
handleOpenFolder,
isInPipeline,
isLoading,
supportBatchUpload,
}: FileListProps) => {
const [inputValue, setInputValue] = useState(keywords)
const { run: updateKeywordsWithDebounce, cancel: cancelKeywordsUpdate } = useDebounceFn(
(keywords: string) => {
updateKeywords(keywords)
},
{ wait: 500 },
)
const handleResetKeywords = () => {
cancelKeywordsUpdate()
setInputValue('')
resetKeywords()
}
const handleSearchValueChange = (keywords: string) => {
if (!keywords) {
handleResetKeywords()
return
}
setInputValue(keywords)
updateKeywordsWithDebounce(keywords)
}
return (
<div className="flex h-100 flex-col overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg shadow-xs shadow-shadow-shadow-3">
<Header
breadcrumbs={breadcrumbs}
inputValue={inputValue}
keywords={keywords}
bucket={bucket}
isInPipeline={isInPipeline}
onSearchValueChange={handleSearchValueChange}
searchResultsLength={searchResultsLength}
/>
<List
fileList={fileList}
selectedFileIds={selectedFileIds}
keywords={keywords}
handleResetKeywords={handleResetKeywords}
handleOpenFolder={handleOpenFolder}
handleSelectFile={handleSelectFile}
isLoading={isLoading}
supportBatchUpload={supportBatchUpload}
/>
</div>
)
}
export default FileList
@@ -0,0 +1,167 @@
import type { ComponentProps } from 'react'
import type { OnlineDriveFile } from '@/models/pipeline'
import { act, render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { OnlineDriveFileType } from '@/models/pipeline'
import { createDataSourceStore } from '../../../../store'
import { DataSourceContext } from '../../../../store/provider'
import List from '../index'
const files: OnlineDriveFile[] = [
{ id: 'first', name: 'First.pdf', type: OnlineDriveFileType.file, size: 1024 },
{ id: 'second', name: 'Second.txt', type: OnlineDriveFileType.file, size: 512 },
{ id: 'folder', name: 'Archive', type: OnlineDriveFileType.folder },
]
const observers: Observer[] = []
class Observer implements IntersectionObserver {
readonly root = null
readonly rootMargin = '100px'
readonly scrollMargin = '0px'
readonly thresholds = [0]
observe = vi.fn()
unobserve = vi.fn()
disconnect = vi.fn()
takeRecords = () => []
private callback: IntersectionObserverCallback
constructor(callback: IntersectionObserverCallback) {
this.callback = callback
observers.push(this)
}
intersect(visible: boolean) {
const target = this.observe.mock.lastCall![0] as Element
const rect = target.getBoundingClientRect()
this.callback(
[
{
isIntersecting: visible,
intersectionRatio: visible ? 1 : 0,
target,
time: 0,
rootBounds: null,
boundingClientRect: rect,
intersectionRect: rect,
},
],
this,
)
}
}
function setup(overrides: Partial<ComponentProps<typeof List>> = {}) {
const store = createDataSourceStore()
const props = {
fileList: files,
selectedFileIds: ['first'],
keywords: '',
isLoading: false,
supportBatchUpload: true,
handleResetKeywords: vi.fn(),
handleSelectFile: vi.fn(),
handleOpenFolder: vi.fn(),
...overrides,
}
const view = render(
<DataSourceContext value={store}>
<List {...props} />
</DataSourceContext>,
)
return {
...view,
store,
props,
update: (next: Partial<typeof props>) =>
view.rerender(
<DataSourceContext value={store}>
<List {...props} {...next} />
</DataSourceContext>,
),
}
}
beforeEach(() => {
observers.length = 0
vi.stubGlobal('IntersectionObserver', Observer)
})
afterEach(() => vi.unstubAllGlobals())
it.each([true, false])('reflects and updates selected files in batch=%s mode', async (multiple) => {
const user = userEvent.setup()
const { props, update } = setup({ supportBatchUpload: multiple })
const role = multiple ? 'checkbox' : 'radio'
expect(screen.getByRole(role, { name: 'First.pdf' })).toBeChecked()
expect(screen.getByRole(role, { name: 'Second.txt' })).not.toBeChecked()
await user.click(screen.getByRole(role, { name: 'Second.txt' }))
expect(props.handleSelectFile).toHaveBeenCalledExactlyOnceWith(files[1])
expect(props.handleOpenFolder).not.toHaveBeenCalled()
update({ selectedFileIds: ['second'] })
expect(screen.getByRole(role, { name: 'First.pdf' })).not.toBeChecked()
expect(screen.getByRole(role, { name: 'Second.txt' })).toBeChecked()
})
it('opens a folder without selecting it', async () => {
const user = userEvent.setup()
const { props } = setup()
await user.click(screen.getByText('Archive'))
expect(props.handleOpenFolder).toHaveBeenCalledExactlyOnceWith(files[2])
expect(props.handleSelectFile).not.toHaveBeenCalled()
})
it('distinguishes an empty folder from a failed search and allows resetting the search', async () => {
const user = userEvent.setup()
const { update, props } = setup({ fileList: [], isLoading: true })
expect(screen.queryByText('datasetPipeline.onlineDrive.emptyFolder')).not.toBeInTheDocument()
update({ fileList: [], isLoading: false })
expect(screen.getByText('datasetPipeline.onlineDrive.emptyFolder')).toBeInTheDocument()
update({ fileList: [], keywords: 'missing', isLoading: false })
expect(screen.getByText('datasetPipeline.onlineDrive.emptySearchResult')).toBeInTheDocument()
expect(screen.queryByText('datasetPipeline.onlineDrive.emptyFolder')).not.toBeInTheDocument()
await user.click(
screen.getByRole('button', { name: 'datasetPipeline.onlineDrive.resetKeywords' }),
)
expect(props.handleResetKeywords).toHaveBeenCalledOnce()
})
it('keeps loaded files visible while loading more and displays the appended page', () => {
const { update } = setup({ isLoading: true })
expect(screen.getByRole('checkbox', { name: 'First.pdf' })).toBeInTheDocument()
expect(screen.getByRole('status', { name: 'appApi.loading' })).toBeInTheDocument()
update({
isLoading: false,
fileList: [...files, { id: 'third', name: 'Third.pdf', type: OnlineDriveFileType.file }],
})
expect(screen.queryByRole('status', { name: 'appApi.loading' })).not.toBeInTheDocument()
expect(screen.getByRole('checkbox', { name: 'Third.pdf' })).toBeInTheDocument()
})
it.each([
{
visible: true,
truncated: true,
loading: false,
expected: { cursor: 'next', bucket: 'archive' },
},
{ visible: false, truncated: true, loading: false, expected: {} },
{ visible: true, truncated: false, loading: false, expected: {} },
{ visible: true, truncated: true, loading: true, expected: {} },
])(
'loads another page only when visible, truncated, and idle: %j',
({ visible, truncated, loading, expected }) => {
const { store } = setup({ isLoading: loading })
store.getState().isTruncated.current = truncated
store.getState().currentNextPageParametersRef.current = { cursor: 'next', bucket: 'archive' }
act(() => observers[0]!.intersect(visible))
expect(store.getState().nextPageParameters).toEqual(expected)
},
)
it('replaces the observer when loading changes and disconnects the current observer on unmount', () => {
const { update, unmount, store } = setup({ isLoading: true })
const first = observers[0]!
update({ isLoading: false })
expect(first.disconnect).toHaveBeenCalledOnce()
const current = observers[1]!
store.getState().isTruncated.current = true
store.getState().currentNextPageParametersRef.current = { cursor: 'latest' }
act(() => current.intersect(true))
expect(store.getState().nextPageParameters).toEqual({ cursor: 'latest' })
unmount()
expect(current.disconnect).toHaveBeenCalledOnce()
})
@@ -0,0 +1,77 @@
import type { OnlineDriveFile } from '@/models/pipeline'
import { RadioGroup } from '@langgenius/dify-ui/radio-group'
import { render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import Item from '../item'
describe('Item', () => {
const makeFile = (
type: OnlineDriveFile['type'],
name = 'test.pdf',
size = 1024,
): OnlineDriveFile => ({
id: 'f-1',
name,
type,
size,
})
const defaultProps = {
file: makeFile('file'),
isSelected: false,
onSelect: vi.fn(),
onOpen: vi.fn(),
}
beforeEach(() => {
vi.clearAllMocks()
})
it('should render radio for file type in single choice mode', () => {
render(
<RadioGroup aria-label="Files" value={defaultProps.file.id}>
<Item {...defaultProps} isMultipleChoice={false} />
</RadioGroup>,
)
expect(screen.getByRole('radio', { name: 'test.pdf' })).toBeInTheDocument()
})
it('should not render checkbox for bucket type', () => {
render(<Item {...defaultProps} file={makeFile('bucket', 'my-bucket')} />)
expect(screen.queryByRole('checkbox')).not.toBeInTheDocument()
})
it('should call onOpen for folder click', async () => {
const user = userEvent.setup()
const file = makeFile('folder', 'my-folder')
render(<Item {...defaultProps} file={file} />)
await user.click(screen.getByText('my-folder'))
expect(defaultProps.onOpen).toHaveBeenCalledWith(file)
})
it('should call onSelect for file click', async () => {
const user = userEvent.setup()
render(<Item {...defaultProps} />)
await user.click(screen.getByText('test.pdf'))
expect(defaultProps.onSelect).toHaveBeenCalledWith(defaultProps.file)
})
it('should call onSelect once without bubbling to row click when checkbox is clicked', async () => {
const user = userEvent.setup()
render(<Item {...defaultProps} />)
await user.click(screen.getByRole('checkbox', { name: 'test.pdf' }))
expect(defaultProps.onSelect).toHaveBeenCalledTimes(1)
expect(defaultProps.onSelect).toHaveBeenCalledWith(defaultProps.file)
expect(defaultProps.onOpen).not.toHaveBeenCalled()
})
it('should not call handlers when disabled', async () => {
const user = userEvent.setup()
render(<Item {...defaultProps} disabled={true} />)
await user.click(screen.getByText('test.pdf'))
expect(defaultProps.onSelect).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,79 @@
import { describe, expect, it } from 'vite-plus/test'
import { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader/types'
import { getFileExtension, getFileType } from '../utils'
describe('getFileExtension', () => {
it('should return extension for normal file', () => {
expect(getFileExtension('test.pdf')).toBe('pdf')
})
it('should return lowercase extension', () => {
expect(getFileExtension('test.PDF')).toBe('pdf')
})
it('should return last extension for multiple dots', () => {
expect(getFileExtension('my.file.name.txt')).toBe('txt')
})
it('should return empty string for no extension', () => {
expect(getFileExtension('noext')).toBe('')
})
it('should return empty string for empty string', () => {
expect(getFileExtension('')).toBe('')
})
it('should return empty string for dotfile with no extension', () => {
expect(getFileExtension('.gitignore')).toBe('')
})
})
describe('getFileType', () => {
it('should return pdf for .pdf files', () => {
expect(getFileType('doc.pdf')).toBe(FileAppearanceTypeEnum.pdf)
})
it('should return markdown for .md files', () => {
expect(getFileType('readme.md')).toBe(FileAppearanceTypeEnum.markdown)
})
it('should return markdown for .mdx files', () => {
expect(getFileType('page.mdx')).toBe(FileAppearanceTypeEnum.markdown)
})
it('should return excel for .xlsx files', () => {
expect(getFileType('data.xlsx')).toBe(FileAppearanceTypeEnum.excel)
})
it('should return excel for .csv files', () => {
expect(getFileType('data.csv')).toBe(FileAppearanceTypeEnum.excel)
})
it('should return word for .docx files', () => {
expect(getFileType('doc.docx')).toBe(FileAppearanceTypeEnum.word)
})
it('should return ppt for .pptx files', () => {
expect(getFileType('slides.pptx')).toBe(FileAppearanceTypeEnum.ppt)
})
it('should return code for .html files', () => {
expect(getFileType('page.html')).toBe(FileAppearanceTypeEnum.code)
})
it('should return code for .json files', () => {
expect(getFileType('config.json')).toBe(FileAppearanceTypeEnum.code)
})
it('should return gif for .gif files', () => {
expect(getFileType('animation.gif')).toBe(FileAppearanceTypeEnum.gif)
})
it('should return custom for unknown extension', () => {
expect(getFileType('file.xyz')).toBe(FileAppearanceTypeEnum.custom)
})
it('should return custom for no extension', () => {
expect(getFileType('noext')).toBe(FileAppearanceTypeEnum.custom)
})
})
@@ -0,0 +1,16 @@
import * as React from 'react'
import { useTranslation } from 'react-i18next'
const EmptyFolder = () => {
const { t } = useTranslation(['datasetPipeline'])
return (
<div className="flex size-full items-center justify-center rounded-[10px] bg-background-section px-1 py-1.5">
<span className="system-xs-regular text-text-tertiary">
{t(($) => $['onlineDrive.emptyFolder'], { ns: 'datasetPipeline' })}
</span>
</div>
)
}
export default React.memo(EmptyFolder)
@@ -0,0 +1,32 @@
import { Button } from '@langgenius/dify-ui/button'
import * as React from 'react'
import { useTranslation } from 'react-i18next'
type EmptySearchResultProps = {
onResetKeywords: () => void
}
const EmptySearchResult = ({
onResetKeywords,
}: EmptySearchResultProps & {
className?: string
}) => {
const { t } = useTranslation(['datasetPipeline'])
return (
<div className="flex size-full flex-col items-center justify-center gap-y-2 rounded-[10px] bg-background-section p-6">
<span
aria-hidden
className="i-custom-vender-knowledge-search-menu size-8 text-text-tertiary"
/>
<div className="system-sm-regular text-text-secondary">
{t(($) => $['onlineDrive.emptySearchResult'], { ns: 'datasetPipeline' })}
</div>
<Button variant="secondary-accent" size="small" onClick={onResetKeywords}>
<span>{t(($) => $['onlineDrive.resetKeywords'], { ns: 'datasetPipeline' })}</span>
</Button>
</div>
)
}
export default React.memo(EmptySearchResult)
@@ -0,0 +1,49 @@
import { cn } from '@langgenius/dify-ui/cn'
import * as React from 'react'
import { useMemo } from 'react'
import FileTypeIcon from '@/app/components/base/file-uploader/file-type-icon'
import { OnlineDriveFileType } from '@/models/pipeline'
import { getFileType } from './utils'
type FileIconProps = {
type: OnlineDriveFileType
fileName: string
size?: 'sm' | 'md' | 'lg' | 'xl'
className?: string
}
const FileIcon = ({ type, fileName, size = 'md', className }: FileIconProps) => {
const fileType = useMemo(() => {
if (type === OnlineDriveFileType.bucket || type === OnlineDriveFileType.folder) return 'custom'
return getFileType(fileName)
}, [type, fileName])
if (type === OnlineDriveFileType.bucket) {
return (
<span
aria-hidden
className={cn(
'i-custom-public-knowledge-online-drive-buckets-blue h-5.25 w-5',
cn('size-4.5', className),
)}
/>
)
}
if (type === OnlineDriveFileType.folder) {
return (
<span
aria-hidden
className={cn(
'i-custom-public-knowledge-online-drive-folder h-4.75 w-5',
cn('size-4.5', className),
)}
/>
)
}
return <FileTypeIcon size={size} type={fileType} className={cn('size-4.5', className)} />
}
export default React.memo(FileIcon)
@@ -0,0 +1,118 @@
import type { OnlineDriveFile } from '@/models/pipeline'
import { RadioGroup } from '@langgenius/dify-ui/radio-group'
import { RiLoader2Line } from '@remixicon/react'
import * as React from 'react'
import { useEffect, useRef } from 'react'
import { useTranslation } from 'react-i18next'
import { LoadingPlaceholder } from '@/app/components/base/loading-placeholder'
import { useDataSourceStore } from '../../../store'
import EmptyFolder from './empty-folder'
import EmptySearchResult from './empty-search-result'
import Item from './item'
type FileListProps = {
fileList: OnlineDriveFile[]
selectedFileIds: string[]
keywords: string
isLoading: boolean
supportBatchUpload: boolean
handleResetKeywords: () => void
handleSelectFile: (file: OnlineDriveFile) => void
handleOpenFolder: (file: OnlineDriveFile) => void
}
const List = ({
fileList,
selectedFileIds,
keywords,
handleResetKeywords,
handleSelectFile,
handleOpenFolder,
isLoading,
supportBatchUpload,
}: FileListProps) => {
const { t } = useTranslation(['appApi', 'datasetPipeline'])
const anchorRef = useRef<HTMLDivElement>(null)
const observerRef = useRef<IntersectionObserver>(null)
const dataSourceStore = useDataSourceStore()
useEffect(() => {
if (anchorRef.current) {
observerRef.current = new IntersectionObserver(
(entries) => {
const { setNextPageParameters, currentNextPageParametersRef, isTruncated } =
dataSourceStore.getState()
if (entries[0]!.isIntersecting && isTruncated.current && !isLoading)
setNextPageParameters(currentNextPageParametersRef.current)
},
{
rootMargin: '100px',
},
)
observerRef.current.observe(anchorRef.current)
}
return () => observerRef.current?.disconnect()
}, [anchorRef, isLoading, dataSourceStore])
const isAllLoading = isLoading && fileList.length === 0 && keywords.length === 0
const isPartialLoading = isLoading && fileList.length > 0
const isEmptyFolder = !isLoading && fileList.length === 0 && keywords.length === 0
const isSearchResultEmpty = !isLoading && fileList.length === 0 && keywords.length > 0
const selectedFileId = selectedFileIds[0]
const handleRadioChange = (fileId: string) => {
const selectedFile = fileList.find((file) => file.id === fileId)
if (selectedFile) handleSelectFile(selectedFile)
}
const fileItems = fileList.map((file) => {
const isSelected = selectedFileIds.includes(file.id)
return (
<Item
key={file.id}
file={file}
isSelected={isSelected}
onSelect={handleSelectFile}
onOpen={handleOpenFolder}
isMultipleChoice={supportBatchUpload}
/>
)
})
return (
<div className="grow overflow-hidden p-1 pt-0">
{isAllLoading && <LoadingPlaceholder className="h-full" />}
{isEmptyFolder && <EmptyFolder />}
{isSearchResultEmpty && <EmptySearchResult onResetKeywords={handleResetKeywords} />}
{fileList.length > 0 && (
<div className="flex h-full flex-col gap-y-px overflow-y-auto rounded-[10px] bg-background-section px-1 py-1.5">
{supportBatchUpload ? (
fileItems
) : (
<RadioGroup
aria-label={t(($) => $['onlineDrive.breadcrumbs.allFiles'], {
ns: 'datasetPipeline',
})}
value={selectedFileId}
onValueChange={handleRadioChange}
className="contents"
>
{fileItems}
</RadioGroup>
)}
{isPartialLoading && (
<div
className="flex items-center justify-center py-2"
role="status"
aria-live="polite"
aria-label={t(($) => $.loading, { ns: 'appApi' })}
>
<RiLoader2Line className="size-4 text-text-tertiary" />
</div>
)}
<div ref={anchorRef} className="h-0" />
</div>
)}
</div>
)
}
export default React.memo(List)
@@ -0,0 +1,116 @@
import type { OnlineDriveFile } from '@/models/pipeline'
import { Checkbox } from '@langgenius/dify-ui/checkbox'
import { Popover, PopoverContent, PopoverTrigger } from '@langgenius/dify-ui/popover'
import { Radio } from '@langgenius/dify-ui/radio-group'
import * as React from 'react'
import { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import { formatFileSize } from '@/utils/format'
import FileIcon from './file-icon'
type ItemProps = {
file: OnlineDriveFile
isSelected: boolean
disabled?: boolean
isMultipleChoice?: boolean
onSelect: (file: OnlineDriveFile) => void
onOpen: (file: OnlineDriveFile) => void
}
const Item = ({
file,
isSelected,
disabled = false,
isMultipleChoice = true,
onSelect,
onOpen,
}: ItemProps) => {
const { t } = useTranslation(['datasetPipeline'])
const { name, type, size } = file
const isBucket = type === 'bucket'
const isFolder = type === 'folder'
const disabledTip = t(($) => $['onlineDrive.notSupportedFileType'], { ns: 'datasetPipeline' })
const handleCheckboxSelect = useCallback(() => {
onSelect(file)
}, [file, onSelect])
const handleClickItem = useCallback(
(e: React.MouseEvent<HTMLDivElement>) => {
e.stopPropagation()
if (disabled) return
if (isBucket || isFolder) {
onOpen(file)
return
}
onSelect(file)
},
[disabled, file, isBucket, isFolder, onOpen, onSelect],
)
return (
<div
className="flex cursor-pointer items-center gap-2 rounded-md px-2 py-0.75 hover:bg-state-base-hover"
onClick={handleClickItem}
>
{!isBucket && isMultipleChoice && (
<span onClick={(event) => event.stopPropagation()}>
<Checkbox
className="shrink-0"
disabled={disabled}
checked={isSelected}
aria-label={name}
onCheckedChange={() => handleCheckboxSelect()}
/>
</span>
)}
{!isBucket && !isMultipleChoice && (
<span onClick={(event) => event.stopPropagation()}>
<Radio className="shrink-0" disabled={disabled} value={file.id} aria-label={name} />
</span>
)}
{disabled ? (
<Popover>
<PopoverTrigger
openOnHover
aria-label={disabledTip}
className="flex grow items-center gap-x-1 overflow-hidden border-0 bg-transparent p-0 py-0.5 text-left opacity-30"
>
<FileIcon type={type} fileName={name} className="shrink-0 transform-gpu" />
<span className="grow truncate system-sm-medium text-text-secondary" title={name}>
{name}
</span>
{!isFolder && typeof size === 'number' && (
<span className="shrink-0 system-xs-regular text-text-tertiary">
{formatFileSize(size)}
</span>
)}
</PopoverTrigger>
<PopoverContent
side="top"
align="end"
className="px-3 py-2 system-xs-regular text-text-tertiary"
>
{disabledTip}
</PopoverContent>
</Popover>
) : (
<div className="flex grow items-center gap-x-1 overflow-hidden py-0.5">
<FileIcon type={type} fileName={name} className="shrink-0 transform-gpu" />
<span className="grow truncate system-sm-medium text-text-secondary" title={name}>
{name}
</span>
{!isFolder && typeof size === 'number' && (
<span className="shrink-0 system-xs-regular text-text-tertiary">
{formatFileSize(size)}
</span>
)}
</div>
)}
</div>
)
}
export default React.memo(Item)
@@ -0,0 +1,41 @@
import { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader/types'
import { FILE_EXTS } from '@/app/components/base/prompt-editor/constants'
export const getFileExtension = (fileName: string): string => {
if (!fileName) return ''
const parts = fileName.split('.')
if (parts.length <= 1 || (parts[0] === '' && parts.length === 2)) return ''
return parts[parts.length - 1]!.toLowerCase()
}
export const getFileType = (fileName: string) => {
const extension = getFileExtension(fileName)
if (extension === 'gif') return FileAppearanceTypeEnum.gif
if (FILE_EXTS.image!.includes(extension.toUpperCase())) return FileAppearanceTypeEnum.image
if (FILE_EXTS.video!.includes(extension.toUpperCase())) return FileAppearanceTypeEnum.video
if (FILE_EXTS.audio!.includes(extension.toUpperCase())) return FileAppearanceTypeEnum.audio
if (extension === 'html' || extension === 'htm' || extension === 'xml' || extension === 'json')
return FileAppearanceTypeEnum.code
if (extension === 'pdf') return FileAppearanceTypeEnum.pdf
if (extension === 'md' || extension === 'markdown' || extension === 'mdx')
return FileAppearanceTypeEnum.markdown
if (extension === 'xlsx' || extension === 'xls' || extension === 'csv')
return FileAppearanceTypeEnum.excel
if (extension === 'docx' || extension === 'doc') return FileAppearanceTypeEnum.word
if (extension === 'pptx' || extension === 'ppt') return FileAppearanceTypeEnum.ppt
if (FILE_EXTS.document!.includes(extension.toUpperCase())) return FileAppearanceTypeEnum.document
return FileAppearanceTypeEnum.custom
}
@@ -0,0 +1,243 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { OnlineDriveFile } from '@/models/pipeline'
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
import { useQuery } from '@tanstack/react-query'
import { produce } from 'immer'
import { useQueryState } from 'nuqs'
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import { useShallow } from 'zustand/react/shallow'
import {
settingsQueryParamName,
settingsQueryParser,
} from '@/app/components/header/account-setting/query-params'
import { toast } from '@/app/notifications'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useDocLink } from '@/context/i18n'
import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline'
import { ssePost } from '@/service/base'
import { consoleQuery } from '@/service/console'
import Header from '../base/header'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
import FileList from './file-list'
import { convertOnlineDriveData } from './utils'
type OnlineDriveProps = {
nodeId: string
nodeData: DataSourceNodeType
onCredentialChange: (credentialId: string) => void
isInPipeline?: boolean
supportBatchUpload?: boolean
}
const OnlineDrive = ({
nodeId,
nodeData,
isInPipeline = false,
supportBatchUpload = true,
onCredentialChange,
}: OnlineDriveProps) => {
const docLink = useDocLink()
const [isInitialMount, setIsInitialMount] = useState(true)
const pipelineId = useDatasetDetailContextWithSelector((s) => s.dataset?.pipeline_id)
const [, setSettingsDestination] = useQueryState(settingsQueryParamName, settingsQueryParser)
const {
nextPageParameters,
breadcrumbs,
prefix,
keywords,
bucket,
selectedFileIds,
onlineDriveFileList,
currentCredentialId,
} = useDataSourceStoreWithSelector(
useShallow((state) => ({
nextPageParameters: state.nextPageParameters,
breadcrumbs: state.breadcrumbs,
prefix: state.prefix,
keywords: state.keywords,
bucket: state.bucket,
selectedFileIds: state.selectedFileIds,
onlineDriveFileList: state.onlineDriveFileList,
currentCredentialId: state.currentCredentialId,
})),
)
const dataSourceStore = useDataSourceStore()
const [isLoading, setIsLoading] = useState(false)
const isLoadingRef = useRef(false)
const { data: dataSourceAuth } = useQuery(
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
input: { params: { provider_id: `${nodeData.plugin_id}/${nodeData.provider_name}` } },
}),
)
const datasourceNodeRunURL = !isInPipeline
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
const getOnlineDriveFiles = useCallback(async () => {
if (isLoadingRef.current) return
const { nextPageParameters, prefix, bucket, onlineDriveFileList, currentCredentialId } =
dataSourceStore.getState()
setIsLoading(true)
isLoadingRef.current = true
ssePost(
datasourceNodeRunURL,
{
body: {
inputs: {
prefix: prefix[prefix.length - 1],
bucket,
next_page_parameters: nextPageParameters,
max_keys: 30,
},
datasource_type: DatasourceType.onlineDrive,
credential_id: currentCredentialId,
},
},
{
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
const {
setOnlineDriveFileList,
isTruncated,
currentNextPageParametersRef,
setHasBucket,
} = dataSourceStore.getState()
const {
fileList: newFileList,
isTruncated: newIsTruncated,
nextPageParameters: newNextPageParameters,
hasBucket: newHasBucket,
} = convertOnlineDriveData(documentsData.data, breadcrumbs, bucket)
setOnlineDriveFileList([...onlineDriveFileList, ...newFileList])
isTruncated.current = newIsTruncated
currentNextPageParametersRef.current = newNextPageParameters
setHasBucket(newHasBucket)
setIsLoading(false)
isLoadingRef.current = false
},
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
toast.error(error.error)
setIsLoading(false)
isLoadingRef.current = false
},
},
)
}, [dataSourceStore, datasourceNodeRunURL, breadcrumbs])
useEffect(() => {
if (!currentCredentialId) return
if (isInitialMount) {
// Only fetch files on initial mount if fileList is empty
if (onlineDriveFileList.length === 0) getOnlineDriveFiles()
setIsInitialMount(false)
} else {
getOnlineDriveFiles()
}
}, [nextPageParameters, prefix, bucket, currentCredentialId])
const filteredOnlineDriveFileList = useMemo(() => {
if (keywords)
return onlineDriveFileList.filter((file) =>
file.name.toLowerCase().includes(keywords.toLowerCase()),
)
return onlineDriveFileList
}, [onlineDriveFileList, keywords])
const updateKeywords = useCallback(
(keywords: string) => {
const { setKeywords } = dataSourceStore.getState()
setKeywords(keywords)
},
[dataSourceStore],
)
const resetKeywords = useCallback(() => {
const { setKeywords } = dataSourceStore.getState()
setKeywords('')
}, [dataSourceStore])
const handleSelectFile = useCallback(
(file: OnlineDriveFile) => {
const { selectedFileIds, setSelectedFileIds } = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.bucket) return
const newSelectedFileList = produce(selectedFileIds, (draft) => {
if (draft.includes(file.id)) {
const index = draft.indexOf(file.id)
draft.splice(index, 1)
} else {
if (!supportBatchUpload && draft.length >= 1) return
draft.push(file.id)
}
})
setSelectedFileIds(newSelectedFileList)
},
[dataSourceStore, supportBatchUpload],
)
const handleOpenFolder = useCallback(
(file: OnlineDriveFile) => {
const {
breadcrumbs,
prefix,
setBreadcrumbs,
setPrefix,
setBucket,
setOnlineDriveFileList,
setSelectedFileIds,
} = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.file) return
setOnlineDriveFileList([])
if (file.type === OnlineDriveFileType.bucket) {
setBucket(file.name)
} else {
setSelectedFileIds([])
const newBreadcrumbs = produce(breadcrumbs, (draft) => {
draft.push(file.name)
})
const newPrefix = produce(prefix, (draft) => {
draft.push(file.id)
})
setBreadcrumbs(newBreadcrumbs)
setPrefix(newPrefix)
}
},
[dataSourceStore],
)
const handleSetting = useCallback(() => {
setSettingsDestination('data-source')
}, [setSettingsDestination])
return (
<div className="flex flex-col gap-y-2">
<Header
docTitle="Docs"
docLink={docLink('/use-dify/knowledge/knowledge-pipeline/authorize-data-source')}
onClickConfiguration={handleSetting}
pluginName={nodeData.datasource_label}
currentCredentialId={currentCredentialId}
onCredentialChange={onCredentialChange}
credentials={dataSourceAuth?.result || []}
/>
<FileList
fileList={filteredOnlineDriveFileList}
selectedFileIds={selectedFileIds}
breadcrumbs={breadcrumbs}
keywords={keywords}
bucket={bucket}
resetKeywords={resetKeywords}
updateKeywords={updateKeywords}
searchResultsLength={filteredOnlineDriveFileList.length}
handleSelectFile={handleSelectFile}
handleOpenFolder={handleOpenFolder}
isInPipeline={isInPipeline}
isLoading={isLoading}
supportBatchUpload={supportBatchUpload}
/>
</div>
)
}
export default OnlineDrive
@@ -0,0 +1,64 @@
import type { OnlineDriveFile } from '@/models/pipeline'
import type { OnlineDriveData } from '@/types/pipeline'
import { OnlineDriveFileType } from '@/models/pipeline'
const isFile = (type: 'file' | 'folder'): boolean => {
return type === 'file'
}
export const isBucketListInitiation = (
data: OnlineDriveData[],
prefix: string[],
bucket: string,
): boolean => {
if (bucket || prefix.length > 0) return false
const hasBucket = data.every((item) => !!item.bucket)
return (
hasBucket &&
(data.length > 1 || (data.length === 1 && !!data[0]!.bucket && data[0]!.files.length === 0))
)
}
export const convertOnlineDriveData = (
data: OnlineDriveData[],
prefix: string[],
bucket: string,
): {
fileList: OnlineDriveFile[]
isTruncated: boolean
nextPageParameters: Record<string, unknown>
hasBucket: boolean
} => {
const fileList: OnlineDriveFile[] = []
let isTruncated = false
let nextPageParameters: Record<string, unknown> = {}
let hasBucket = false
if (data.length === 0) return { fileList, isTruncated, nextPageParameters, hasBucket }
if (isBucketListInitiation(data, prefix, bucket)) {
data.forEach((item) => {
fileList.push({
id: item.bucket,
name: item.bucket,
type: OnlineDriveFileType.bucket,
})
})
hasBucket = true
} else {
data[0]!.files.forEach((file) => {
const { id, name, size, type } = file
const isFileType = isFile(type)
fileList.push({
id,
name,
size: isFileType ? size : undefined,
type: isFileType ? OnlineDriveFileType.file : OnlineDriveFileType.folder,
})
})
isTruncated = data[0]!.is_truncated ?? false
nextPageParameters = data[0]!.next_page_parameters ?? {}
hasBucket = !!data[0]!.bucket
}
return { fileList, isTruncated, nextPageParameters, hasBucket }
}
@@ -0,0 +1,43 @@
import type { CommonShape } from './slices/common'
import type { LocalFileSliceShape } from './slices/local-file'
import type { OnlineDocumentSliceShape } from './slices/online-document'
import type { OnlineDriveSliceShape } from './slices/online-drive'
import type { WebsiteCrawlSliceShape } from './slices/website-crawl'
import { use } from 'react'
import { createStore, useStore } from 'zustand'
import { DataSourceContext } from './provider'
import { createCommonSlice } from './slices/common'
import { createLocalFileSlice } from './slices/local-file'
import { createOnlineDocumentSlice } from './slices/online-document'
import { createOnlineDriveSlice } from './slices/online-drive'
import { createWebsiteCrawlSlice } from './slices/website-crawl'
export type DataSourceShape = CommonShape &
LocalFileSliceShape &
OnlineDocumentSliceShape &
WebsiteCrawlSliceShape &
OnlineDriveSliceShape
export const createDataSourceStore = () => {
return createStore<DataSourceShape>((...args) => ({
...createCommonSlice(...args),
...createLocalFileSlice(...args),
...createOnlineDocumentSlice(...args),
...createWebsiteCrawlSlice(...args),
...createOnlineDriveSlice(...args),
}))
}
export const useDataSourceStoreWithSelector = <T>(selector: (state: DataSourceShape) => T): T => {
const store = use(DataSourceContext)
if (!store) throw new Error('Missing DataSourceContext.Provider in the tree')
return useStore(store, selector)
}
export const useDataSourceStore = () => {
const store = use(DataSourceContext)
if (!store) throw new Error('Missing DataSourceContext.Provider in the tree')
return store
}
@@ -0,0 +1,23 @@
import { useRefWithInit } from '@langgenius/dify-ui/use-ref-with-init'
import { createContext } from 'react'
import { createDataSourceStore } from './'
type DataSourceStoreApi = ReturnType<typeof createDataSourceStore>
type DataSourceContextType = DataSourceStoreApi | null
export const DataSourceContext = createContext<DataSourceContextType>(null)
type DataSourceProviderProps = {
children: React.ReactNode
}
const DataSourceProvider = ({ children }: DataSourceProviderProps) => {
const storeRef = useRefWithInit(createDataSourceStore)
return (
<DataSourceContext.Provider value={storeRef.current}>{children}</DataSourceContext.Provider>
)
}
export default DataSourceProvider
@@ -0,0 +1,19 @@
import type { StateCreator } from 'zustand'
export type CommonShape = {
currentNodeIdRef: React.RefObject<string>
currentCredentialId: string
setCurrentCredentialId: (credentialId: string) => void
currentCredentialIdRef: React.RefObject<string>
}
export const createCommonSlice: StateCreator<CommonShape> = (set) => {
return {
currentNodeIdRef: { current: '' },
currentCredentialId: '',
setCurrentCredentialId: (credentialId: string) => {
set({ currentCredentialId: credentialId })
},
currentCredentialIdRef: { current: '' },
}
}
@@ -0,0 +1,29 @@
import type { StateCreator } from 'zustand'
import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets'
export type LocalFileSliceShape = {
localFileList: FileItem[]
setLocalFileList: (fileList: FileItem[]) => void
currentLocalFile: File | undefined
setCurrentLocalFile: (file: File | undefined) => void
previewLocalFileRef: React.RefObject<DocumentItem | undefined>
}
export const createLocalFileSlice: StateCreator<LocalFileSliceShape> = (set, get) => {
return {
localFileList: [],
setLocalFileList: (fileList: FileItem[]) => {
set(() => ({
localFileList: fileList,
}))
const { previewLocalFileRef } = get()
previewLocalFileRef.current = fileList[0]?.file as DocumentItem
},
currentLocalFile: undefined,
setCurrentLocalFile: (file: File | undefined) =>
set(() => ({
currentLocalFile: file,
})),
previewLocalFileRef: { current: undefined },
}
}
@@ -0,0 +1,50 @@
import type { StateCreator } from 'zustand'
import type { DataSourceNotionWorkspace, NotionPage } from '@/models/common'
export type OnlineDocumentSliceShape = {
documentsData: DataSourceNotionWorkspace[]
setDocumentsData: (documentData: DataSourceNotionWorkspace[]) => void
searchValue: string
setSearchValue: (searchValue: string) => void
onlineDocuments: NotionPage[]
setOnlineDocuments: (documents: NotionPage[]) => void
currentDocument: NotionPage | undefined
setCurrentDocument: (document: NotionPage | undefined) => void
selectedPagesId: Set<string>
setSelectedPagesId: (selectedPagesId: Set<string>) => void
previewOnlineDocumentRef: React.RefObject<NotionPage | undefined>
}
export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set, get) => {
return {
documentsData: [],
setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) =>
set(() => ({
documentsData,
})),
searchValue: '',
setSearchValue: (searchValue: string) =>
set(() => ({
searchValue,
})),
onlineDocuments: [],
setOnlineDocuments: (documents: NotionPage[]) => {
set(() => ({
onlineDocuments: documents,
}))
const { previewOnlineDocumentRef } = get()
previewOnlineDocumentRef.current = documents[0]
},
currentDocument: undefined,
setCurrentDocument: (document: NotionPage | undefined) =>
set(() => ({
currentDocument: document,
})),
selectedPagesId: new Set(),
setSelectedPagesId: (selectedPagesId: Set<string>) =>
set(() => ({
selectedPagesId,
})),
previewOnlineDocumentRef: { current: undefined },
}
}
@@ -0,0 +1,76 @@
import type { StateCreator } from 'zustand'
import type { OnlineDriveFile } from '@/models/pipeline'
export type OnlineDriveSliceShape = {
breadcrumbs: string[]
setBreadcrumbs: (breadcrumbs: string[]) => void
prefix: string[]
setPrefix: (prefix: string[]) => void
keywords: string
setKeywords: (keywords: string) => void
selectedFileIds: string[]
setSelectedFileIds: (selectedFileIds: string[]) => void
onlineDriveFileList: OnlineDriveFile[]
setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => void
bucket: string
setBucket: (bucket: string) => void
nextPageParameters: Record<string, unknown>
currentNextPageParametersRef: React.RefObject<Record<string, unknown>>
setNextPageParameters: (nextPageParameters: Record<string, unknown>) => void
isTruncated: React.RefObject<boolean>
previewOnlineDriveFileRef: React.RefObject<OnlineDriveFile | undefined>
hasBucket: boolean
setHasBucket: (hasBucket: boolean) => void
}
export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set, get) => {
return {
breadcrumbs: [],
setBreadcrumbs: (breadcrumbs: string[]) =>
set(() => ({
breadcrumbs,
})),
prefix: [],
setPrefix: (prefix: string[]) =>
set(() => ({
prefix,
})),
keywords: '',
setKeywords: (keywords: string) =>
set(() => ({
keywords,
})),
selectedFileIds: [],
setSelectedFileIds: (selectedFileIds: string[]) => {
set(() => ({
selectedFileIds,
}))
const id = selectedFileIds[0]
const { onlineDriveFileList, previewOnlineDriveFileRef } = get()
previewOnlineDriveFileRef.current = onlineDriveFileList.find((file) => file.id === id)
},
onlineDriveFileList: [],
setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) =>
set(() => ({
onlineDriveFileList,
})),
bucket: '',
setBucket: (bucket: string) =>
set(() => ({
bucket,
})),
nextPageParameters: {},
currentNextPageParametersRef: { current: {} },
setNextPageParameters: (nextPageParameters: Record<string, unknown>) =>
set(() => ({
nextPageParameters,
})),
isTruncated: { current: false },
previewOnlineDriveFileRef: { current: undefined },
hasBucket: false,
setHasBucket: (hasBucket: boolean) =>
set(() => ({
hasBucket,
})),
}
}
@@ -0,0 +1,51 @@
import type { StateCreator } from 'zustand'
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
import { CrawlStep } from '@/models/datasets'
export type WebsiteCrawlSliceShape = {
websitePages: CrawlResultItem[]
setWebsitePages: (pages: CrawlResultItem[]) => void
currentWebsite: CrawlResultItem | undefined
setCurrentWebsite: (website: CrawlResultItem | undefined) => void
crawlResult: CrawlResult | undefined
setCrawlResult: (result: CrawlResult | undefined) => void
step: CrawlStep
setStep: (step: CrawlStep) => void
previewIndex: number
setPreviewIndex: (index: number) => void
previewWebsitePageRef: React.RefObject<CrawlResultItem | undefined>
}
export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (set, get) => {
return {
websitePages: [],
setWebsitePages: (pages: CrawlResultItem[]) => {
set(() => ({
websitePages: pages,
}))
const { previewWebsitePageRef } = get()
previewWebsitePageRef.current = pages[0]
},
currentWebsite: undefined,
setCurrentWebsite: (website: CrawlResultItem | undefined) =>
set(() => ({
currentWebsite: website,
})),
crawlResult: undefined,
setCrawlResult: (result: CrawlResult | undefined) =>
set(() => ({
crawlResult: result,
})),
step: CrawlStep.init,
setStep: (step: CrawlStep) =>
set(() => ({
step,
})),
previewIndex: -1,
setPreviewIndex: (index: number) =>
set(() => ({
previewIndex: index,
})),
previewWebsitePageRef: { current: undefined },
}
}
@@ -0,0 +1,310 @@
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { CrawlResultItem } from '@/models/datasets'
import type {
DataSourceNodeCompletedResponse,
DataSourceNodeErrorResponse,
DataSourceNodeProcessingResponse,
} from '@/types/pipeline'
import { act, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { BlockEnum } from '@/app/components/workflow/types'
import { CrawlStep } from '@/models/datasets'
import { DatasourceType } from '@/models/pipeline'
import { consoleQuery } from '@/service/console'
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
import { createNuqsTestRenderer } from '@/test/nuqs-testing'
import { createPipelineVariable } from '@/test/pipeline-variables'
import { createDataSourceStore } from '../../store'
import { DataSourceContext } from '../../store/provider'
import WebsiteCrawl from '../index'
const { ssePost, parameters } = vi.hoisted(() => ({ ssePost: vi.fn(), parameters: vi.fn() }))
vi.mock('@/service/base', async (importOriginal) => ({
...(await importOriginal<typeof import('@/service/base')>()),
ssePost,
}))
vi.mock('@/context/dataset-detail', () => ({
useDatasetDetailContextWithSelector: (
select: (state: { dataset: { pipeline_id: string } }) => unknown,
) => select({ dataset: { pipeline_id: 'pipeline-123' } }),
}))
const onUrlUpdate = vi.fn()
const render = createNuqsTestRenderer(renderWithConsoleQuery, () => ({ onUrlUpdate }))
const nodeData: DataSourceNodeType = {
type: BlockEnum.DataSource,
title: 'Crawler',
desc: '',
provider_type: DatasourceType.websiteCrawl,
plugin_id: 'plugin-123',
provider_name: 'crawler',
datasource_name: 'crawler',
datasource_label: 'Crawler',
datasource_parameters: {},
datasource_configurations: {},
}
const credentials: DatasourceCredentialResponse[] = [
{
id: 'credential-1',
name: 'First credential',
avatar_url: '',
type: 'oauth2',
is_default: false,
credential: {},
},
{
id: 'credential-2',
name: 'Second credential',
avatar_url: '',
type: 'oauth2',
is_default: false,
credential: {},
},
]
const pages: CrawlResultItem[] = [
{
title: 'First page',
source_url: 'https://example.com/1',
markdown: '# First',
description: 'First',
},
{
title: 'Second page',
source_url: 'https://example.com/2',
markdown: '# Second',
description: 'Second',
},
]
function setup({
credential = 'credential-1',
single = false,
draft = false,
loading = false,
} = {}) {
parameters.mockImplementation(() =>
loading
? new Promise(() => {})
: Promise.resolve({
variables: [
createPipelineVariable({
variable: 'url',
label: 'URL',
default_value: 'https://example.com',
}),
],
}),
)
const store = createDataSourceStore()
store.setState({ currentCredentialId: credential })
const queryClient = createConsoleQueryClient()
queryClient.setQueryData(
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
input: { params: { provider_id: 'plugin-123/crawler' } },
}).queryKey,
{ result: credential ? credentials : [] },
)
const workflows = consoleQuery.rag.pipelines.byPipelineId.workflows
const options = (
draft ? workflows.draft : workflows.published
).preProcessing.parameters.get.queryOptions({
input: { params: { pipeline_id: 'pipeline-123' }, query: { node_id: 'node-1' } },
})
if (loading)
queryClient.setQueryDefaults(options.queryKey, { queryFn: () => new Promise(() => {}) })
else
queryClient.setQueryData(options.queryKey, {
variables: [
createPipelineVariable({
variable: 'url',
label: 'URL',
default_value: 'https://example.com',
}),
],
})
const onCredentialChange = vi.fn((id: string) => {
store.setState({
currentCredentialId: id,
step: CrawlStep.init,
websitePages: [],
crawlResult: undefined,
})
})
render(
<DataSourceContext value={store}>
<WebsiteCrawl
nodeId="node-1"
nodeData={nodeData}
onCredentialChange={onCredentialChange}
supportBatchUpload={!single}
isInPipeline={draft}
/>
</DataSourceContext>,
{ queryClient },
)
return { store, onCredentialChange }
}
function callbacks(): {
onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => void
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => void
onDataSourceNodeError: (data: DataSourceNodeErrorResponse) => void
} {
return ssePost.mock.lastCall![2]
}
const runButton = () => screen.getByRole('button', { name: 'datasetCreation.stepOne.website.run' })
function complete() {
act(() =>
callbacks().onDataSourceNodeCompleted({
event: 'datasource_completed',
time_consuming: 2.5,
data: pages,
}),
)
}
beforeEach(() => vi.clearAllMocks())
it.each([true, false])(
'runs the draft=%s workflow, displays streamed progress and selects completed pages',
async (draft) => {
const user = userEvent.setup()
const { store } = setup({ draft })
await waitFor(() => expect(runButton()).toBeEnabled())
const input = screen.getByRole('textbox', { name: /URL/ })
await user.clear(input)
await user.type(input, 'https://example.com/docs')
await waitFor(() => expect(runButton()).toBeEnabled())
await user.click(runButton())
await waitFor(() =>
expect(ssePost).toHaveBeenCalledExactlyOnceWith(
`/rag/pipelines/pipeline-123/workflows/${draft ? 'draft' : 'published'}/datasource/nodes/node-1/run`,
{
body: {
inputs: { url: 'https://example.com/docs' },
credential_id: 'credential-1',
datasource_type: DatasourceType.websiteCrawl,
response_mode: 'streaming',
},
},
expect.any(Object),
),
)
act(() =>
callbacks().onDataSourceNodeProcessing({
event: 'datasource_processing',
total: 2,
completed: 1,
}),
)
expect(screen.getByText(/totalPageScraped/)).toHaveTextContent('1/2')
await user.click(
screen.getByRole('button', { name: 'datasetCreation.stepOne.website.running' }),
)
expect(ssePost).toHaveBeenCalledOnce()
complete()
expect(store.getState().step).toBe(CrawlStep.finished)
expect(store.getState().websitePages).toEqual(pages)
expect(screen.getByRole('checkbox', { name: /First page/ })).toBeChecked()
expect(screen.getByRole('checkbox', { name: /Second page/ })).toBeChecked()
expect(
screen.queryAllByRole('button', { name: 'datasetCreation.stepOne.website.preview' }),
).toHaveLength(draft ? 0 : 2)
},
)
it.each([{ credential: '' }, { loading: true }])(
'prevents crawling when credentials or parameters are unavailable: %j',
async (options) => {
const user = userEvent.setup()
setup(options)
expect(runButton()).toBeDisabled()
await user.click(runButton())
expect(ssePost).not.toHaveBeenCalled()
},
)
it('keeps only one page selected in single-upload mode', async () => {
const user = userEvent.setup()
const { store } = setup({ single: true })
await waitFor(() => expect(runButton()).toBeEnabled())
await user.click(runButton())
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
complete()
expect(store.getState().websitePages).toEqual([pages[0]])
await user.click(screen.getByRole('radio', { name: /Second page/ }))
expect(store.getState().websitePages).toEqual([pages[1]])
expect(screen.getByRole('radio', { name: /First page/ })).not.toBeChecked()
})
it('deselects a page and previews it without changing the remaining selection', async () => {
const user = userEvent.setup()
const { store } = setup()
await waitFor(() => expect(runButton()).toBeEnabled())
await user.click(runButton())
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
complete()
await user.click(screen.getByRole('checkbox', { name: /Second page/ }))
await user.click(
screen.getAllByRole('button', { name: 'datasetCreation.stepOne.website.preview' })[1]!,
)
expect(store.getState().websitePages).toEqual([pages[0]])
expect(store.getState().currentWebsite).toEqual(pages[1])
expect(store.getState().previewIndex).toBe(1)
})
it.each(['Crawler unavailable', ''])(
'shows a stream error and clears it after a successful retry: %s',
async (message) => {
const user = userEvent.setup()
const { store } = setup()
await waitFor(() => expect(runButton()).toBeEnabled())
await user.click(runButton())
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
act(() => callbacks().onDataSourceNodeError({ event: 'datasource_error', error: message }))
const error = message || 'datasetCreation.stepOne.website.unknownError'
expect(screen.getByText(error)).toBeInTheDocument()
expect(store.getState().websitePages).toEqual([])
await waitFor(() => expect(runButton()).toBeEnabled())
await user.click(runButton())
await waitFor(() => expect(ssePost).toHaveBeenCalledTimes(2))
complete()
expect(screen.queryByText(error)).not.toBeInTheDocument()
expect(store.getState().websitePages).toEqual(pages)
},
)
it('uses the changed credential on the next run and opens configuration', async () => {
const user = userEvent.setup()
const { onCredentialChange } = setup()
await waitFor(() => expect(runButton()).toBeEnabled())
await user.click(runButton())
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
act(() =>
callbacks().onDataSourceNodeError({
event: 'datasource_error',
error: 'Old credential failed',
}),
)
await user.click(screen.getByRole('button', { name: /First credential/ }))
await user.click(screen.getByText('Second credential'))
expect(onCredentialChange).toHaveBeenCalledExactlyOnceWith('credential-2')
await waitFor(() => expect(runButton()).toBeEnabled())
await user.click(runButton())
await waitFor(() => expect(ssePost).toHaveBeenCalledTimes(2))
expect(ssePost.mock.lastCall![1].body.credential_id).toBe('credential-2')
expect(screen.queryByText('Old credential failed')).not.toBeInTheDocument()
await user.click(screen.getByRole('button', { name: /^datasetPipeline\.configurationTip/ }))
await waitFor(() =>
expect(onUrlUpdate.mock.lastCall![0].searchParams.get('settings')).toBe('data-source'),
)
})
vi.mock('@/service/console', async (importOriginal) => {
const actual = await importOriginal<typeof import('@/service/console')>()
const { withConsoleResponses } = await import('@/test/console/service-responses')
return {
...actual,
consoleQuery: withConsoleResponses(actual.consoleQuery, {
'rag.pipelines.byPipelineId.workflows.published.preProcessing.parameters.get': parameters,
'rag.pipelines.byPipelineId.workflows.draft.preProcessing.parameters.get': parameters,
}),
}
})
@@ -0,0 +1,62 @@
import type { CrawlResultItem } from '@/models/datasets'
import { render, screen } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { useState } from 'react'
import CrawledResult from '../crawled-result'
const pages: CrawlResultItem[] = ['First', 'Second', 'Third'].map((title, index) => ({
title,
source_url: `https://example.com/${index}`,
markdown: title,
description: title,
}))
function Harness({ multiple = true, onChange = vi.fn(), onPreview = vi.fn() }) {
const [selected, setSelected] = useState([pages[0]!])
return (
<CrawledResult
list={pages}
checkedList={selected}
usedTime={12.3}
isMultipleChoice={multiple}
showPreview
onPreview={onPreview}
onSelectedChange={(next) => {
setSelected(next)
onChange(next)
}}
/>
)
}
it('adds and removes individual pages and supports selecting and resetting all', async () => {
const user = userEvent.setup()
const onChange = vi.fn()
render(<Harness onChange={onChange} />)
await user.click(screen.getByRole('checkbox', { name: 'Second https://example.com/1' }))
expect(onChange).toHaveBeenLastCalledWith([pages[0], pages[1]])
await user.click(screen.getByRole('checkbox', { name: 'First https://example.com/0' }))
expect(onChange).toHaveBeenLastCalledWith([pages[1]])
await user.click(
screen.getByRole('checkbox', { name: 'datasetCreation.stepOne.website.selectAll' }),
)
expect(onChange).toHaveBeenLastCalledWith(pages)
await user.click(
screen.getByRole('checkbox', { name: 'datasetCreation.stepOne.website.resetAll' }),
)
expect(onChange).toHaveBeenLastCalledWith([])
})
it('replaces the selected page in single-choice mode and previews independently', async () => {
const user = userEvent.setup()
const onChange = vi.fn()
const onPreview = vi.fn()
render(<Harness multiple={false} onChange={onChange} onPreview={onPreview} />)
expect(screen.queryByRole('checkbox')).not.toBeInTheDocument()
await user.click(screen.getByRole('radio', { name: 'Second https://example.com/1' }))
expect(onChange).toHaveBeenCalledWith([pages[1]])
await user.click(
screen.getAllByRole('button', { name: 'datasetCreation.stepOne.website.preview' })[2]!,
)
expect(onPreview).toHaveBeenCalledWith(pages[2], 2)
expect(onChange).toHaveBeenCalledOnce()
})
@@ -0,0 +1,52 @@
'use client'
import { Checkbox } from '@langgenius/dify-ui/checkbox'
import { cn } from '@langgenius/dify-ui/cn'
import { Infotip, InfotipContent, InfotipTrigger } from '@langgenius/dify-ui/infotip'
import { useId } from 'react'
type CheckboxWithLabelProps = {
className?: string
isChecked: boolean
onChange: (isChecked: boolean) => void
label: string
labelClassName?: string
tooltip?: string
}
export default function CheckboxWithLabel({
className = '',
isChecked,
onChange,
label,
labelClassName,
tooltip,
}: CheckboxWithLabelProps) {
const labelId = useId()
return (
<div className={cn('flex items-center', className)}>
<label className="flex min-w-0 cursor-pointer items-center">
<Checkbox checked={isChecked} onCheckedChange={(checked) => onChange(checked)} />
<span
id={labelId}
className={cn(
'ml-2 min-w-0 text-left system-sm-medium text-text-secondary',
labelClassName,
)}
>
{label}
</span>
</label>
<div className="ml-1 flex min-w-0 items-center">
{tooltip && (
<Infotip>
<InfotipTrigger aria-labelledby={labelId} />
<InfotipContent aria-labelledby={labelId} className="w-50">
{tooltip}
</InfotipContent>
</Infotip>
)}
</div>
</div>
)
}
@@ -0,0 +1,85 @@
'use client'
import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
import { Button } from '@langgenius/dify-ui/button'
import { Checkbox } from '@langgenius/dify-ui/checkbox'
import { cn } from '@langgenius/dify-ui/cn'
import { Radio } from '@langgenius/dify-ui/radio-group'
import * as React from 'react'
import { useTranslation } from 'react-i18next'
type CrawledResultItemProps = {
payload: CrawlResultItemType
isChecked: boolean
onCheckChange: (checked: boolean) => void
isPreview: boolean
showPreview: boolean
onPreview: () => void
isMultipleChoice?: boolean
}
const CrawledResultItem = ({
payload,
isChecked,
onCheckChange,
isPreview,
onPreview,
showPreview,
isMultipleChoice = true,
}: CrawledResultItemProps) => {
const { t } = useTranslation(['datasetCreation'])
return (
<div
className={cn(
'relative flex gap-x-2 rounded-lg p-2',
isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover',
)}
>
{isMultipleChoice ? (
<label className="flex min-w-0 grow cursor-pointer gap-x-2">
<Checkbox
className="shrink-0"
checked={isChecked}
onCheckedChange={(checked) => onCheckChange(checked)}
/>
<div className="flex min-w-0 grow flex-col gap-y-0.5">
<div className="truncate system-sm-medium text-text-secondary" title={payload.title}>
{payload.title}
</div>
<div
className="truncate system-xs-regular text-text-tertiary"
title={payload.source_url}
>
{payload.source_url}
</div>
</div>
</label>
) : (
<label className="flex min-w-0 grow cursor-pointer gap-x-2">
<Radio className="shrink-0" value={payload.source_url} />
<div className="flex min-w-0 grow flex-col gap-y-0.5">
<div className="truncate system-sm-medium text-text-secondary" title={payload.title}>
{payload.title}
</div>
<div
className="truncate system-xs-regular text-text-tertiary"
title={payload.source_url}
>
{payload.source_url}
</div>
</div>
</label>
)}
{showPreview && (
<Button
size="small"
onClick={onPreview}
className="top-2 right-2 hidden px-1.5 system-xs-medium-uppercase group-hover:absolute group-hover:block"
>
{t(($) => $['stepOne.website.preview'], { ns: 'datasetCreation' })}
</Button>
)}
</div>
)
}
export default React.memo(CrawledResultItem)
@@ -0,0 +1,136 @@
'use client'
import type { CrawlResultItem } from '@/models/datasets'
import { cn } from '@langgenius/dify-ui/cn'
import { RadioGroup } from '@langgenius/dify-ui/radio-group'
import * as React from 'react'
import { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import CheckboxWithLabel from './checkbox-with-label'
import CrawledResultItem from './crawled-result-item'
const I18N_PREFIX = 'stepOne.website'
type CrawledResultProps = {
className?: string
previewIndex?: number
list: CrawlResultItem[]
checkedList: CrawlResultItem[]
onSelectedChange: (selected: CrawlResultItem[]) => void
onPreview?: (payload: CrawlResultItem, index: number) => void
showPreview?: boolean
usedTime: number
isMultipleChoice?: boolean
}
const CrawledResult = ({
className = '',
previewIndex,
list,
checkedList,
onSelectedChange,
usedTime,
onPreview,
showPreview = false,
isMultipleChoice = true,
}: CrawledResultProps) => {
const { t } = useTranslation(['datasetCreation'])
const isCheckAll = checkedList.length === list.length
const handleCheckedAll = useCallback(() => {
if (!isCheckAll) onSelectedChange(list)
else onSelectedChange([])
}, [isCheckAll, list, onSelectedChange])
const handleItemCheckChange = useCallback(
(item: CrawlResultItem) => {
return (checked: boolean) => {
if (checked) {
if (isMultipleChoice) onSelectedChange([...checkedList, item])
else onSelectedChange([item])
} else {
onSelectedChange(
checkedList.filter((checkedItem) => checkedItem.source_url !== item.source_url),
)
}
}
},
[checkedList, onSelectedChange, isMultipleChoice],
)
const handlePreview = useCallback(
(index: number) => {
if (!onPreview) return
onPreview(list[index]!, index)
},
[list, onPreview],
)
const selectedSourceUrl = checkedList[0]?.source_url
const handleRadioChange = useCallback(
(sourceUrl: string) => {
const selectedItem = list.find((item) => item.source_url === sourceUrl)
if (selectedItem) onSelectedChange([selectedItem])
},
[list, onSelectedChange],
)
const resultItems = list.map((item, index) => (
<CrawledResultItem
key={item.source_url}
payload={item}
isChecked={checkedList.some((checkedItem) => checkedItem.source_url === item.source_url)}
onCheckChange={handleItemCheckChange(item)}
isPreview={index === previewIndex}
onPreview={handlePreview.bind(null, index)}
showPreview={showPreview}
isMultipleChoice={isMultipleChoice}
/>
))
return (
<div className={cn('flex flex-col gap-y-2', className)}>
<div className="pt-2 system-sm-medium text-text-primary">
{t(($) => $[`${I18N_PREFIX}.scrapTimeInfo`], {
ns: 'datasetCreation',
total: list.length,
time: usedTime.toFixed(1),
})}
</div>
<div className="overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg">
{isMultipleChoice && (
<div className="flex items-center px-4 py-2">
<CheckboxWithLabel
isChecked={isCheckAll}
onChange={handleCheckedAll}
label={
isCheckAll
? t(($) => $[`${I18N_PREFIX}.resetAll`], { ns: 'datasetCreation' })
: t(($) => $[`${I18N_PREFIX}.selectAll`], { ns: 'datasetCreation' })
}
/>
</div>
)}
{isMultipleChoice ? (
<div className="flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2">
{resultItems}
</div>
) : (
<RadioGroup
aria-label={t(($) => $[`${I18N_PREFIX}.scrapTimeInfo`], {
ns: 'datasetCreation',
total: list.length,
time: usedTime.toFixed(1),
})}
value={selectedSourceUrl}
onValueChange={handleRadioChange}
className="flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2"
>
{resultItems}
</RadioGroup>
)}
</div>
</div>
)
}
export default React.memo(CrawledResult)
@@ -0,0 +1,85 @@
'use client'
import { cn } from '@langgenius/dify-ui/cn'
import * as React from 'react'
import { useTranslation } from 'react-i18next'
type CrawlingProps = {
className?: string
crawledNum: number
totalNum: number
}
type BlockProps = {
className?: string
}
type ItemProps = {
firstLineWidth: string
secondLineWidth: string
}
const Block = React.memo(({ className }: BlockProps) => {
return <div className={cn('bg-text-quaternary opacity-20', className)} />
})
const Item = React.memo(({ firstLineWidth, secondLineWidth }: ItemProps) => {
return (
<div className="flex gap-x-2 px-2 py-1.25">
<div className="py-0.5">
<Block className="size-4 rounded-sm" />
</div>
<div className="flex grow flex-col">
<div className="flex h-5 w-full items-center">
<Block className={cn('h-2.5 rounded-xs', firstLineWidth)} />
</div>
<div className="flex h-4.5 w-full items-center">
<Block className={cn('h-1.5 rounded-xs', secondLineWidth)} />
</div>
</div>
</div>
)
})
const Crawling = ({ className = '', crawledNum, totalNum }: CrawlingProps) => {
const { t } = useTranslation(['datasetCreation'])
const itemsConfig = [
{
firstLineWidth: 'w-[35%]',
secondLineWidth: 'w-[50%]',
},
{
firstLineWidth: 'w-[40%]',
secondLineWidth: 'w-[45%]',
},
{
firstLineWidth: 'w-[30%]',
secondLineWidth: 'w-[36%]',
},
]
return (
<div className={cn('mt-2 flex flex-col gap-y-2 pt-2', className)}>
<div className="system-sm-medium text-text-primary">
{t(($) => $['stepOne.website.totalPageScraped'], { ns: 'datasetCreation' })} {crawledNum}/
{totalNum}
</div>
<div className="overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg">
<div className="flex items-center gap-x-2 px-4 py-2">
<Block className="size-4 rounded-sm" />
<Block className="h-2.5 w-14 rounded-xs" />
</div>
<div className="flex flex-col gap-px border-t border-divider-subtle bg-background-default-subtle p-2">
{itemsConfig.map((item, index) => (
<Item
key={index}
firstLineWidth={item.firstLineWidth}
secondLineWidth={item.secondLineWidth}
/>
))}
</div>
</div>
</div>
)
}
export default React.memo(Crawling)
@@ -0,0 +1,18 @@
import { Alert, AlertDescription, AlertTitle } from '@langgenius/dify-ui/alert'
import * as React from 'react'
type ErrorMessageProps = {
className?: string
title: string
errorMsg?: string
}
const ErrorMessage = ({ className, title, errorMsg }: ErrorMessageProps) => {
return (
<Alert type="error" className={className}>
<AlertTitle>{title}</AlertTitle>
{errorMsg && <AlertDescription>{errorMsg}</AlertDescription>}
</Alert>
)
}
export default React.memo(ErrorMessage)
@@ -0,0 +1,54 @@
import { render, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { toast } from '@/app/notifications'
import { CrawlStep } from '@/models/datasets'
import { createPipelineVariable } from '@/test/pipeline-variables'
import Options from '../index'
vi.mock('@/app/notifications', () => ({ toast: { error: vi.fn() } }))
const variables = [
createPipelineVariable({ variable: 'url', label: 'URL', default_value: 'https://example.com' }),
]
const runName = 'datasetCreation.stepOne.website.run'
it('rejects an empty required field and runs with the corrected URL', async () => {
const user = userEvent.setup()
const onSubmit = vi.fn()
render(<Options variables={variables} step={CrawlStep.init} onSubmit={onSubmit} />)
const input = screen.getByRole('textbox', { name: /URL/ })
expect(input).toHaveValue('https://example.com')
await user.clear(input)
await user.click(screen.getByRole('button', { name: runName }))
await waitFor(() => expect(toast.error).toHaveBeenCalledWith('"url" URL is required'))
expect(onSubmit).not.toHaveBeenCalled()
await user.type(input, 'https://example.com/docs')
await user.click(screen.getByRole('button', { name: runName }))
await waitFor(() =>
expect(onSubmit).toHaveBeenCalledExactlyOnceWith({ url: 'https://example.com/docs' }),
)
})
it('preserves edited options across crawling and reopens them for another run', async () => {
const user = userEvent.setup()
const props = { variables, onSubmit: vi.fn() }
const { rerender } = render(<Options {...props} step={CrawlStep.init} />)
const input = screen.getByRole('textbox', { name: /URL/ })
await user.clear(input)
await user.type(input, 'https://example.com/changed')
rerender(<Options {...props} step={CrawlStep.running} />)
expect(screen.queryByRole('textbox')).not.toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'datasetCreation.stepOne.website.running' }))
expect(props.onSubmit).not.toHaveBeenCalled()
rerender(<Options {...props} step={CrawlStep.finished} />)
expect(screen.queryByRole('textbox')).not.toBeInTheDocument()
rerender(<Options {...props} step={CrawlStep.init} />)
expect(screen.getByRole('textbox', { name: /URL/ })).toHaveValue('https://example.com/changed')
})
it('does not start a crawl when the caller disables running', async () => {
const user = userEvent.setup()
const onSubmit = vi.fn()
render(<Options variables={variables} step={CrawlStep.init} onSubmit={onSubmit} runDisabled />)
await user.click(screen.getByRole('button', { name: runName }))
expect(onSubmit).not.toHaveBeenCalled()
})
@@ -0,0 +1,122 @@
import type { RAGPipelineVariables } from '@/models/pipeline'
import { Button } from '@langgenius/dify-ui/button'
import { cn } from '@langgenius/dify-ui/cn'
import { RiPlayLargeLine } from '@remixicon/react'
import { useBoolean } from 'ahooks'
import { useEffect, useId, useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import { useAppForm } from '@/app/components/base/form'
import BaseField from '@/app/components/base/form/form-scenarios/base/field'
import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
import {
useConfigurations,
useInitialData,
} from '@/app/components/rag-pipeline/hooks/use-input-fields'
import { toast } from '@/app/notifications'
import { CrawlStep } from '@/models/datasets'
const I18N_PREFIX = 'stepOne.website'
type OptionsProps = {
variables: RAGPipelineVariables
step: CrawlStep
runDisabled?: boolean
onSubmit: (data: Record<string, any>) => void
}
const Options = ({ variables, step, runDisabled, onSubmit }: OptionsProps) => {
const { t } = useTranslation(['datasetCreation'])
const runButtonLabelId = useId()
const initialData = useInitialData(variables)
const configurations = useConfigurations(variables)
const schema = useMemo(() => {
return generateZodSchema(configurations)
}, [configurations])
const form = useAppForm({
defaultValues: initialData,
validators: {
onSubmit: ({ value }) => {
const result = schema.safeParse(value)
if (!result.success) {
const issues = result.error.issues
const firstIssue = issues[0]
const errorMessage = `"${firstIssue!.path.join('.')}" ${firstIssue!.message}`
toast.error(errorMessage)
return errorMessage
}
return undefined
},
},
onSubmit: ({ value }) => {
onSubmit(value)
},
})
const [fold, { toggle: foldToggle, setTrue: foldHide, setFalse: foldShow }] = useBoolean(false)
useEffect(() => {
// When the step change
if (step !== CrawlStep.init) foldHide()
else foldShow()
}, [step])
const isRunning = step === CrawlStep.running
return (
<form
className="w-full"
onSubmit={(e) => {
e.preventDefault()
e.stopPropagation()
form.handleSubmit()
}}
>
<div className="flex items-center gap-x-1 px-4 py-2">
<div
className="flex grow cursor-pointer items-center gap-x-0.5 select-none"
onClick={foldToggle}
>
<span className="system-sm-semibold-uppercase text-text-secondary">
{t(($) => $[`${I18N_PREFIX}.options`], { ns: 'datasetCreation' })}
</span>
<span
aria-hidden
className={cn(
'i-custom-vender-solid-general-arrow-down-round-fill h-4 w-4',
cn('size-4 shrink-0 text-text-quaternary', fold && '-rotate-90'),
)}
/>
</div>
<Button
variant="primary"
onClick={form.handleSubmit}
disabled={runDisabled}
loading={isRunning}
aria-labelledby={runButtonLabelId}
className="shrink-0"
>
<RiPlayLargeLine aria-hidden className="size-4" />
<span id={runButtonLabelId}>
{!isRunning
? t(($) => $[`${I18N_PREFIX}.run`], { ns: 'datasetCreation' })
: t(($) => $[`${I18N_PREFIX}.running`], { ns: 'datasetCreation' })}
</span>
</Button>
</div>
{!fold && (
<div className="flex flex-col gap-3 border-t border-divider-subtle px-4 py-3">
{configurations.map((config, index) => {
const FieldComponent = BaseField({
initialData,
config,
})
return <FieldComponent key={index} form={form} />
})}
</div>
)}
</form>
)
}
export default Options
@@ -0,0 +1,224 @@
'use client'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { CrawlResultItem } from '@/models/datasets'
import type {
DataSourceNodeCompletedResponse,
DataSourceNodeErrorResponse,
DataSourceNodeProcessingResponse,
} from '@/types/pipeline'
import { useQuery } from '@tanstack/react-query'
import { useQueryState } from 'nuqs'
import * as React from 'react'
import { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useShallow } from 'zustand/react/shallow'
import {
settingsQueryParamName,
settingsQueryParser,
} from '@/app/components/header/account-setting/query-params'
import { selectPipelineStepParameters } from '@/app/components/rag-pipeline/utils/query-select'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useDocLink } from '@/context/i18n'
import { CrawlStep } from '@/models/datasets'
import { DatasourceType } from '@/models/pipeline'
import { ssePost } from '@/service/base'
import { consoleQuery } from '@/service/console'
import Header from '../base/header'
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
import CrawledResult from './base/crawled-result'
import Crawling from './base/crawling'
import ErrorMessage from './base/error-message'
import Options from './base/options'
const I18N_PREFIX = 'stepOne.website'
type WebsiteCrawlProps = {
nodeId: string
nodeData: DataSourceNodeType
onCredentialChange: (credentialId: string) => void
isInPipeline?: boolean
supportBatchUpload?: boolean
}
const WebsiteCrawl = ({
nodeId,
nodeData,
isInPipeline = false,
supportBatchUpload = true,
onCredentialChange,
}: WebsiteCrawlProps) => {
const { t } = useTranslation(['datasetCreation'])
const docLink = useDocLink()
const [totalNum, setTotalNum] = useState(0)
const [crawledNum, setCrawledNum] = useState(0)
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
const pipelineId = useDatasetDetailContextWithSelector((s) => s.dataset?.pipeline_id)
const [, setSettingsDestination] = useQueryState(settingsQueryParamName, settingsQueryParser)
const { crawlResult, step, checkedCrawlResult, previewIndex, currentCredentialId } =
useDataSourceStoreWithSelector(
useShallow((state) => ({
crawlResult: state.crawlResult,
step: state.step,
checkedCrawlResult: state.websitePages,
previewIndex: state.previewIndex,
currentCredentialId: state.currentCredentialId,
})),
)
const { data: dataSourceAuth } = useQuery(
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
input: { params: { provider_id: `${nodeData.plugin_id}/${nodeData.provider_name}` } },
}),
)
const dataSourceStore = useDataSourceStore()
const preProcessingParamsOptions = {
input: { params: { pipeline_id: pipelineId! }, query: { node_id: nodeId } },
enabled: !!pipelineId && !!nodeId,
select: selectPipelineStepParameters,
}
const workflows = consoleQuery.rag.pipelines.byPipelineId.workflows
const { data: paramsConfig, isFetching: isFetchingParams } = useQuery(
isInPipeline
? workflows.draft.preProcessing.parameters.get.queryOptions(preProcessingParamsOptions)
: workflows.published.preProcessing.parameters.get.queryOptions(preProcessingParamsOptions),
)
const isInit = step === CrawlStep.init
const isCrawlFinished = step === CrawlStep.finished
const isRunning = step === CrawlStep.running
const showError = isCrawlFinished && crawlErrorMessage
const datasourceNodeRunURL = !isInPipeline
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
const handleCheckedCrawlResultChange = useCallback(
(checkedCrawlResult: CrawlResultItem[]) => {
const { setWebsitePages } = dataSourceStore.getState()
setWebsitePages(checkedCrawlResult)
},
[dataSourceStore],
)
const handlePreview = useCallback(
(website: CrawlResultItem, index: number) => {
const { setCurrentWebsite, setPreviewIndex } = dataSourceStore.getState()
setCurrentWebsite(website)
setPreviewIndex(index)
},
[dataSourceStore],
)
const handleRun = useCallback(
async (value: Record<string, any>) => {
const { setStep, setCrawlResult, currentCredentialId } = dataSourceStore.getState()
setStep(CrawlStep.running)
ssePost(
datasourceNodeRunURL,
{
body: {
inputs: value,
datasource_type: DatasourceType.websiteCrawl,
credential_id: currentCredentialId,
response_mode: 'streaming',
},
},
{
onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => {
setTotalNum(data.total ?? 0)
setCrawledNum(data.completed ?? 0)
},
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => {
const { data: crawlData, time_consuming } = data
const crawlResultData = {
data: crawlData as CrawlResultItem[],
time_consuming: time_consuming ?? 0,
}
setCrawlResult(crawlResultData)
handleCheckedCrawlResultChange(supportBatchUpload ? crawlData : crawlData.slice(0, 1)) // default select the crawl result
setCrawlErrorMessage('')
setStep(CrawlStep.finished)
},
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
setCrawlErrorMessage(
error.error || t(($) => $[`${I18N_PREFIX}.unknownError`], { ns: 'datasetCreation' }),
)
setStep(CrawlStep.finished)
},
},
)
},
[dataSourceStore, datasourceNodeRunURL, handleCheckedCrawlResultChange, supportBatchUpload, t],
)
const handleSubmit = useCallback(
(value: Record<string, any>) => {
handleRun(value)
},
[handleRun],
)
const handleSetting = useCallback(() => {
setSettingsDestination('data-source')
}, [setSettingsDestination])
const handleCredentialChange = useCallback(
(credentialId: string) => {
setCrawledNum(0)
setTotalNum(0)
setCrawlErrorMessage('')
onCredentialChange(credentialId)
},
[onCredentialChange],
)
return (
<div className="flex flex-col">
<Header
docTitle="Docs"
docLink={docLink('/use-dify/knowledge/knowledge-pipeline/authorize-data-source')}
onClickConfiguration={handleSetting}
pluginName={nodeData.datasource_label}
currentCredentialId={currentCredentialId}
onCredentialChange={handleCredentialChange}
credentials={dataSourceAuth?.result || []}
/>
<div className="mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle">
<Options
variables={paramsConfig?.variables || []}
step={step}
runDisabled={!currentCredentialId || isFetchingParams}
onSubmit={handleSubmit}
/>
</div>
{!isInit && (
<div className="relative flex flex-col">
{isRunning && <Crawling crawledNum={crawledNum} totalNum={totalNum} />}
{showError && (
<ErrorMessage
className="mt-2"
title={t(($) => $[`${I18N_PREFIX}.exceptionErrorTitle`], { ns: 'datasetCreation' })}
errorMsg={crawlErrorMessage}
/>
)}
{isCrawlFinished && !showError && (
<CrawledResult
className="mt-2"
list={crawlResult?.data || []}
checkedList={checkedCrawlResult}
onSelectedChange={handleCheckedCrawlResultChange}
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
previewIndex={previewIndex}
onPreview={handlePreview}
showPreview={!isInPipeline}
isMultipleChoice={supportBatchUpload} // only support single choice in test run
/>
)}
</div>
)}
</div>
)
}
export default React.memo(WebsiteCrawl)