chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动
演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
60158935f4
commit
6519682885
15831 files changed
+3063428
No files matched your search
+25
@@ -0,0 +1,25 @@
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import Header from '../header'
|
||||
|
||||
it('links to datasource documentation and opens that plugin configuration', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onClickConfiguration = vi.fn()
|
||||
render(
|
||||
<Header
|
||||
docTitle="Datasource documentation"
|
||||
docLink="https://docs.example.com/source"
|
||||
pluginName="Drive"
|
||||
onClickConfiguration={onClickConfiguration}
|
||||
credentials={[]}
|
||||
currentCredentialId=""
|
||||
onCredentialChange={vi.fn()}
|
||||
/>,
|
||||
)
|
||||
expect(screen.getByRole('link', { name: 'Datasource documentation' })).toHaveAttribute(
|
||||
'href',
|
||||
'https://docs.example.com/source',
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: /datasetPipeline.configurationTip/ }))
|
||||
expect(onClickConfiguration).toHaveBeenCalledOnce()
|
||||
})
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
|
||||
import { render, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import CredentialSelector from '../index'
|
||||
|
||||
const credentials = [
|
||||
{
|
||||
id: 'credential-1',
|
||||
name: 'First credential',
|
||||
avatar_url: 'https://example.com/first.png',
|
||||
type: 'oauth2',
|
||||
is_default: false,
|
||||
},
|
||||
{
|
||||
id: 'credential-2',
|
||||
name: 'Second credential',
|
||||
avatar_url: 'https://example.com/second.png',
|
||||
type: 'oauth2',
|
||||
is_default: false,
|
||||
},
|
||||
] as DatasourceCredentialResponse[]
|
||||
|
||||
describe('CredentialSelector', () => {
|
||||
it('selects the first credential when the current id is invalid', async () => {
|
||||
const onCredentialChange = vi.fn()
|
||||
render(
|
||||
<CredentialSelector
|
||||
currentCredentialId="missing"
|
||||
credentials={credentials}
|
||||
onCredentialChange={onCredentialChange}
|
||||
/>,
|
||||
)
|
||||
|
||||
await waitFor(() => expect(onCredentialChange).toHaveBeenCalledWith('credential-1'))
|
||||
})
|
||||
|
||||
it('selects the visible default credential before falling back to the first one', async () => {
|
||||
const onCredentialChange = vi.fn()
|
||||
const credentialsWithDefault = credentials.map((credential, index) => ({
|
||||
...credential,
|
||||
is_default: index === 1,
|
||||
}))
|
||||
render(
|
||||
<CredentialSelector
|
||||
currentCredentialId="missing"
|
||||
credentials={credentialsWithDefault}
|
||||
onCredentialChange={onCredentialChange}
|
||||
/>,
|
||||
)
|
||||
|
||||
await waitFor(() => expect(onCredentialChange).toHaveBeenCalledWith('credential-2'))
|
||||
})
|
||||
|
||||
it('does not select a fallback for an empty credential list', () => {
|
||||
const onCredentialChange = vi.fn()
|
||||
render(
|
||||
<CredentialSelector
|
||||
currentCredentialId="missing"
|
||||
credentials={[]}
|
||||
onCredentialChange={onCredentialChange}
|
||||
/>,
|
||||
)
|
||||
|
||||
expect(onCredentialChange).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('selects another credential and reflects the controlled value after closing the popup', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onCredentialChange = vi.fn()
|
||||
const { rerender } = render(
|
||||
<CredentialSelector
|
||||
currentCredentialId="credential-1"
|
||||
credentials={credentials}
|
||||
onCredentialChange={onCredentialChange}
|
||||
/>,
|
||||
)
|
||||
const trigger = screen.getByRole('button', { name: /First credential/ })
|
||||
expect(trigger).toHaveAttribute('aria-expanded', 'false')
|
||||
expect(screen.queryByRole('img')).not.toBeInTheDocument()
|
||||
await user.click(trigger)
|
||||
await user.click(await screen.findByText('Second credential'))
|
||||
expect(onCredentialChange).toHaveBeenCalledExactlyOnceWith('credential-2')
|
||||
expect(trigger).toHaveAttribute('aria-expanded', 'false')
|
||||
rerender(
|
||||
<CredentialSelector
|
||||
currentCredentialId="credential-2"
|
||||
credentials={credentials}
|
||||
onCredentialChange={onCredentialChange}
|
||||
/>,
|
||||
)
|
||||
expect(screen.getByRole('button', { name: /Second credential/ })).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { Popover, PopoverContent, PopoverTrigger } from '@langgenius/dify-ui/popover'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import * as React from 'react'
|
||||
import { useCallback, useEffect, useMemo } from 'react'
|
||||
import List from './list'
|
||||
import Trigger from './trigger'
|
||||
|
||||
export type CredentialSelectorProps = {
|
||||
currentCredentialId: string
|
||||
onCredentialChange: (credentialId: string) => void
|
||||
credentials: Array<DatasourceCredentialResponse>
|
||||
}
|
||||
|
||||
const CredentialSelector = ({
|
||||
currentCredentialId,
|
||||
onCredentialChange,
|
||||
credentials,
|
||||
}: CredentialSelectorProps) => {
|
||||
const [open, { set, setFalse }] = useBoolean(false)
|
||||
|
||||
const currentCredential = useMemo(() => {
|
||||
return credentials.find((cred) => cred.id === currentCredentialId)
|
||||
}, [credentials, currentCredentialId])
|
||||
|
||||
useEffect(() => {
|
||||
if (!currentCredential && credentials.length) {
|
||||
const fallbackCredential = credentials.find((credential) => credential.is_default)
|
||||
onCredentialChange((fallbackCredential ?? credentials[0]!).id)
|
||||
}
|
||||
}, [currentCredential, credentials, onCredentialChange])
|
||||
|
||||
const handleCredentialChange = useCallback(
|
||||
(credentialId: string) => {
|
||||
onCredentialChange(credentialId)
|
||||
setFalse()
|
||||
},
|
||||
[onCredentialChange, setFalse],
|
||||
)
|
||||
|
||||
return (
|
||||
<Popover open={open} onOpenChange={set}>
|
||||
<PopoverTrigger
|
||||
nativeButton={false}
|
||||
render={(props, state) => (
|
||||
<div {...props} className={cn('grow overflow-hidden', props.className)}>
|
||||
<Trigger currentCredential={currentCredential} isOpen={state.open} />
|
||||
</div>
|
||||
)}
|
||||
/>
|
||||
<PopoverContent
|
||||
align="start"
|
||||
sideOffset={4}
|
||||
className="border-none bg-transparent shadow-none"
|
||||
>
|
||||
<List
|
||||
currentCredentialId={currentCredentialId}
|
||||
credentials={credentials}
|
||||
onCredentialChange={handleCredentialChange}
|
||||
/>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(CredentialSelector)
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
|
||||
import { RiCheckLine } from '@remixicon/react'
|
||||
import * as React from 'react'
|
||||
import { useCallback } from 'react'
|
||||
import { CredentialIcon } from '@/app/components/datasets/common/credential-icon'
|
||||
|
||||
type ItemProps = {
|
||||
credential: DatasourceCredentialResponse
|
||||
isSelected: boolean
|
||||
onCredentialChange: (credentialId: string) => void
|
||||
}
|
||||
|
||||
const Item = ({ credential, isSelected, onCredentialChange }: ItemProps) => {
|
||||
const { avatar_url, name } = credential
|
||||
|
||||
const handleCredentialChange = useCallback(() => {
|
||||
onCredentialChange(credential.id)
|
||||
}, [credential.id, onCredentialChange])
|
||||
|
||||
return (
|
||||
<div
|
||||
className="flex cursor-pointer items-center gap-x-2 rounded-lg p-2 hover:bg-state-base-hover"
|
||||
onClick={handleCredentialChange}
|
||||
>
|
||||
<CredentialIcon avatarUrl={avatar_url ?? undefined} name={name} size={20} />
|
||||
<span className="grow truncate system-sm-medium text-text-secondary">{name}</span>
|
||||
{isSelected && <RiCheckLine className="size-4 shrink-0 text-text-accent" />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Item)
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
|
||||
import * as React from 'react'
|
||||
import Item from './item'
|
||||
|
||||
type ListProps = {
|
||||
currentCredentialId: string
|
||||
credentials: Array<DatasourceCredentialResponse>
|
||||
onCredentialChange: (credentialId: string) => void
|
||||
}
|
||||
|
||||
const List = ({ currentCredentialId, credentials, onCredentialChange }: ListProps) => {
|
||||
return (
|
||||
<div className="flex w-70 flex-col gap-y-1 rounded-xl border-[0.5px] border-components-panel-border bg-components-panel-bg-blur p-1 shadow-lg shadow-shadow-shadow-5 backdrop-blur-[5px]">
|
||||
{credentials.map((credential) => {
|
||||
const isSelected = credential.id === currentCredentialId
|
||||
return (
|
||||
<Item
|
||||
key={credential.id}
|
||||
credential={credential}
|
||||
isSelected={isSelected}
|
||||
onCredentialChange={onCredentialChange}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(List)
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { RiArrowDownSLine } from '@remixicon/react'
|
||||
import * as React from 'react'
|
||||
import { CredentialIcon } from '@/app/components/datasets/common/credential-icon'
|
||||
|
||||
type TriggerProps = {
|
||||
currentCredential: DatasourceCredentialResponse | undefined
|
||||
isOpen: boolean
|
||||
}
|
||||
|
||||
const Trigger = ({ currentCredential, isOpen }: TriggerProps) => {
|
||||
const { avatar_url, name = '' } = currentCredential || {}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
'flex cursor-pointer items-center gap-x-2 rounded-md p-1 pr-2',
|
||||
isOpen ? 'bg-state-base-hover' : 'hover:bg-state-base-hover',
|
||||
)}
|
||||
>
|
||||
<CredentialIcon avatarUrl={avatar_url ?? undefined} name={name} size={20} />
|
||||
<div className="flex grow items-center gap-x-1 overflow-hidden">
|
||||
<span className="grow truncate system-md-semibold text-text-secondary">{name}</span>
|
||||
<RiArrowDownSLine className="size-4 shrink-0 text-text-secondary" />
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Trigger)
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
import type { CredentialSelectorProps } from './credential-selector'
|
||||
import { IconButton } from '@langgenius/dify-ui/icon-button'
|
||||
import { Separator } from '@langgenius/dify-ui/separator'
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@langgenius/dify-ui/tooltip'
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import CredentialSelector from './credential-selector'
|
||||
|
||||
type HeaderProps = {
|
||||
docTitle: string
|
||||
docLink: string
|
||||
onClickConfiguration?: () => void
|
||||
pluginName: string
|
||||
} & CredentialSelectorProps
|
||||
|
||||
const Header = ({ docTitle, docLink, onClickConfiguration, pluginName, ...rest }: HeaderProps) => {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
const configurationTip = t(($) => $.configurationTip, { ns: 'datasetPipeline', pluginName })
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-x-2">
|
||||
<div className="flex items-center gap-x-1 overflow-hidden">
|
||||
<CredentialSelector {...rest} />
|
||||
<Separator decorative orientation="vertical" className="mx-1 h-3.5" />
|
||||
<Tooltip>
|
||||
<TooltipTrigger
|
||||
render={
|
||||
<IconButton
|
||||
variant="ghost"
|
||||
size="md"
|
||||
className="shrink-0"
|
||||
aria-label={configurationTip}
|
||||
onClick={onClickConfiguration}
|
||||
>
|
||||
<span aria-hidden className="i-ri-equalizer-2-line size-4" />
|
||||
</IconButton>
|
||||
}
|
||||
/>
|
||||
<TooltipContent>{configurationTip}</TooltipContent>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<a
|
||||
className="flex shrink-0 items-center gap-x-1 system-xs-medium text-text-accent"
|
||||
href={docLink}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
<span aria-hidden className="i-ri-book-open-line size-3.5 shrink-0" />
|
||||
<span title={docTitle}>{docTitle}</span>
|
||||
</a>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Header)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
import type { FileResponse } from '@dify/contracts/api/console/files/types.gen'
|
||||
import { screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { toast } from '@/app/notifications'
|
||||
import { AppToaster } from '@/app/notifications/app-toaster'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
|
||||
import { createFileUploadConfigFixture } from '@/test/fixtures/file-upload-config'
|
||||
import { createDataSourceStore } from '../../store'
|
||||
import { DataSourceContext } from '../../store/provider'
|
||||
import LocalFile from '../index'
|
||||
|
||||
const upload = vi.hoisted(() => vi.fn<typeof import('@/service/base').upload>())
|
||||
const supportedFormats = vi.hoisted(() => vi.fn(async () => ({ allowed_extensions: ['pdf'] })))
|
||||
vi.mock('@/service/base', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('@/service/base')>()),
|
||||
upload,
|
||||
}))
|
||||
vi.mock('@/service/console', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@/service/console')>()
|
||||
const { withConsoleResponses } = await import('@/test/console/service-responses')
|
||||
return {
|
||||
...actual,
|
||||
consoleQuery: withConsoleResponses(actual.consoleQuery, {
|
||||
'files.supportType.get': supportedFormats,
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
const setup = ({
|
||||
allowedExtensions = ['pdf'],
|
||||
supportBatchUpload = true,
|
||||
existing = false,
|
||||
} = {}) => {
|
||||
const user = userEvent.setup({ applyAccept: false })
|
||||
const store = createDataSourceStore()
|
||||
const file = Object.assign(new File(['document'], 'Guide.pdf', { type: 'application/pdf' }), {
|
||||
id: 'file-1',
|
||||
})
|
||||
if (existing) store.getState().setLocalFileList([{ fileID: 'local-1', file, progress: 100 }])
|
||||
const queryClient = createConsoleQueryClient()
|
||||
queryClient.setQueryData(
|
||||
consoleQuery.files.upload.get.queryKey(),
|
||||
createFileUploadConfigFixture(),
|
||||
)
|
||||
renderWithConsoleQuery(
|
||||
<DataSourceContext value={store}>
|
||||
<LocalFile allowedExtensions={allowedExtensions} supportBatchUpload={supportBatchUpload} />
|
||||
<AppToaster timeout={0} />
|
||||
</DataSourceContext>,
|
||||
{ queryClient, systemFeatures: { deployment_edition: 'CLOUD' } },
|
||||
)
|
||||
return { user, store, queryClient, file }
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
upload.mockReset().mockResolvedValue({
|
||||
id: 'csv-file',
|
||||
name: 'data.csv',
|
||||
size: 3,
|
||||
extension: 'csv',
|
||||
mime_type: 'text/csv',
|
||||
} satisfies FileResponse)
|
||||
})
|
||||
afterEach(() => toast.dismiss())
|
||||
|
||||
it.each([true, false])(
|
||||
'previews a stored file and applies batch-upload availability (%s)',
|
||||
async (supportBatchUpload) => {
|
||||
const { user, store, file } = setup({ supportBatchUpload, existing: true })
|
||||
await user.click(screen.getByText('Guide.pdf'))
|
||||
expect(store.getState().currentLocalFile).toBe(file)
|
||||
const browse = screen.queryByRole('button', { name: 'datasetCreation.stepOne.uploader.browse' })
|
||||
if (supportBatchUpload) expect(browse).toBeInTheDocument()
|
||||
else expect(browse).not.toBeInTheDocument()
|
||||
},
|
||||
)
|
||||
|
||||
it.each([
|
||||
{ allowedExtensions: ['csv'], accept: '.csv', canUpload: true },
|
||||
{ allowedExtensions: [], accept: '', canUpload: false },
|
||||
])(
|
||||
'preserves the pipeline format override $allowedExtensions after supported formats load',
|
||||
async ({ allowedExtensions, accept, canUpload }) => {
|
||||
const { user, store, queryClient } = setup({ allowedExtensions })
|
||||
const input = screen.getByLabelText('datasetCreation.stepOne.uploader.browse')
|
||||
expect(input).toHaveAttribute('accept', accept)
|
||||
await waitFor(() =>
|
||||
expect(queryClient.getQueryData(consoleQuery.files.supportType.get.queryKey())).toEqual({
|
||||
allowed_extensions: ['pdf'],
|
||||
}),
|
||||
)
|
||||
expect(supportedFormats).toHaveBeenCalledOnce()
|
||||
expect(input).toHaveAttribute('accept', accept)
|
||||
await user.upload(input, new File(['row'], 'data.csv', { type: 'text/csv' }))
|
||||
if (canUpload) {
|
||||
await waitFor(() =>
|
||||
expect(store.getState().localFileList[0]).toMatchObject({
|
||||
file: { id: 'csv-file', name: 'data.csv' },
|
||||
progress: 100,
|
||||
}),
|
||||
)
|
||||
await user.click(screen.getByText('data.csv'))
|
||||
expect(store.getState().currentLocalFile?.id).toBe('csv-file')
|
||||
expect(upload).toHaveBeenCalledOnce()
|
||||
} else {
|
||||
expect(
|
||||
await screen.findByText('datasetCreation.stepOne.uploader.validation.typeError'),
|
||||
).toBeVisible()
|
||||
expect(
|
||||
screen.queryByRole('button', { name: 'datasetCreation.stepOne.uploader.browse' }),
|
||||
).not.toBeInTheDocument()
|
||||
expect(store.getState().localFileList).toEqual([])
|
||||
expect(upload).not.toHaveBeenCalled()
|
||||
}
|
||||
},
|
||||
)
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import type { FileItem } from '@/models/datasets'
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import FileListItem from '../file-list-item'
|
||||
|
||||
it('previews the uploaded file selected by name', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onPreview = vi.fn()
|
||||
const fileItem: FileItem = {
|
||||
fileID: 'file-1',
|
||||
progress: 100,
|
||||
file: Object.assign(new File(['content'], 'guide.pdf', { type: 'application/pdf' }), {
|
||||
id: 'uploaded-1',
|
||||
}),
|
||||
}
|
||||
render(<FileListItem fileItem={fileItem} onPreview={onPreview} onRemove={vi.fn()} />)
|
||||
await user.click(screen.getByText('guide.pdf'))
|
||||
expect(onPreview).toHaveBeenCalledWith(fileItem.file)
|
||||
})
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
import type { RefObject } from 'react'
|
||||
import type { UploadDropzoneProps } from '../upload-dropzone'
|
||||
import { screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { describe, expect, it, vi } from 'vite-plus/test'
|
||||
import { renderWithConsoleQuery as render } from '@/test/console/query-data'
|
||||
import UploadDropzone from '../upload-dropzone'
|
||||
|
||||
const createMockRef = <T,>(): RefObject<T | null> => ({ current: null })
|
||||
|
||||
describe('UploadDropzone', () => {
|
||||
const defaultProps: UploadDropzoneProps = {
|
||||
dropRef: createMockRef<HTMLDivElement>(),
|
||||
dragRef: createMockRef<HTMLDivElement>(),
|
||||
fileUploaderRef: createMockRef<HTMLInputElement>(),
|
||||
dragging: false,
|
||||
supportBatchUpload: true,
|
||||
supportTypesShowNames: 'PDF, DOCX, TXT',
|
||||
fileUploadConfig: {
|
||||
file_size_limit: 15,
|
||||
batch_count_limit: 5,
|
||||
file_upload_limit: 10,
|
||||
},
|
||||
acceptTypes: ['.pdf', '.docx', '.txt'],
|
||||
allowedExtensions: ['pdf', 'docx', 'txt'],
|
||||
onSelectFile: vi.fn(),
|
||||
onFileChange: vi.fn(),
|
||||
}
|
||||
|
||||
it.each(['COMMUNITY', 'CLOUD'] as const)(
|
||||
'shows the applicable upload limits for %s',
|
||||
(edition) => {
|
||||
render(<UploadDropzone {...defaultProps} />, {
|
||||
systemFeatures: { deployment_edition: edition },
|
||||
})
|
||||
const tip = screen.getByText(/datasetCreation.stepOne.uploader.tip/)
|
||||
expect(tip).toHaveTextContent('"size":15')
|
||||
expect(tip).toHaveTextContent('"batchCount":5')
|
||||
expect(tip).toHaveTextContent('"supportTypes":"PDF, DOCX, TXT"')
|
||||
if (edition === 'CLOUD') expect(tip).toHaveTextContent('"totalCount":10')
|
||||
else expect(tip).not.toHaveTextContent('"totalCount"')
|
||||
},
|
||||
)
|
||||
|
||||
it('disables browsing when no file types are allowed', () => {
|
||||
render(<UploadDropzone {...defaultProps} allowedExtensions={[]} />)
|
||||
expect(
|
||||
screen.queryByRole('button', { name: 'datasetCreation.stepOne.uploader.browse' }),
|
||||
).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('forwards browsing and selected files to the upload owner', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onFiles = vi.fn()
|
||||
render(
|
||||
<UploadDropzone
|
||||
{...defaultProps}
|
||||
onFileChange={(event) => onFiles(Array.from(event.target.files ?? []))}
|
||||
/>,
|
||||
)
|
||||
await user.click(
|
||||
screen.getByRole('button', { name: 'datasetCreation.stepOne.uploader.browse' }),
|
||||
)
|
||||
expect(defaultProps.onSelectFile).toHaveBeenCalledOnce()
|
||||
const files = [
|
||||
new File(['a'], 'a.pdf', { type: 'application/pdf' }),
|
||||
new File(['b'], 'b.txt', { type: 'text/plain' }),
|
||||
]
|
||||
await user.upload(document.getElementById('fileUploader') as HTMLInputElement, files)
|
||||
expect(onFiles).toHaveBeenCalledWith(files)
|
||||
})
|
||||
})
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { RiDeleteBinLine, RiErrorWarningFill } from '@remixicon/react'
|
||||
import dynamic from 'next/dynamic'
|
||||
import { useMemo } from 'react'
|
||||
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
|
||||
import { getFileType } from '@/app/components/datasets/common/image-uploader/utils'
|
||||
import useTheme from '@/hooks/use-theme'
|
||||
import { Theme } from '@/types/app'
|
||||
import { formatFileSize } from '@/utils/format'
|
||||
import { PROGRESS_ERROR } from '../constants'
|
||||
|
||||
const SimplePieChart = dynamic(() => import('@/app/components/base/simple-pie-chart'), {
|
||||
ssr: false,
|
||||
})
|
||||
|
||||
type FileListItemProps = {
|
||||
fileItem: FileItem
|
||||
onPreview: (file: File) => void
|
||||
onRemove: (fileID: string) => void
|
||||
}
|
||||
|
||||
const FileListItem = ({ fileItem, onPreview, onRemove }: FileListItemProps) => {
|
||||
const { theme } = useTheme()
|
||||
const chartColor = useMemo(() => (theme === Theme.dark ? '#5289ff' : '#296dff'), [theme])
|
||||
|
||||
const isUploading = fileItem.progress >= 0 && fileItem.progress < 100
|
||||
const isError = fileItem.progress === PROGRESS_ERROR
|
||||
|
||||
const handleClick = () => {
|
||||
onPreview(fileItem.file)
|
||||
}
|
||||
|
||||
const handleRemove = (e: React.MouseEvent) => {
|
||||
e.stopPropagation()
|
||||
onRemove(fileItem.fileID)
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
onClick={handleClick}
|
||||
className={cn(
|
||||
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
|
||||
isError && 'border-state-destructive-border bg-state-destructive-hover',
|
||||
)}
|
||||
>
|
||||
<div className="flex w-12 shrink-0 items-center justify-center">
|
||||
<DocumentFileIcon
|
||||
size="lg"
|
||||
className="shrink-0"
|
||||
name={fileItem.file.name}
|
||||
extension={getFileType(fileItem.file)}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex shrink grow flex-col gap-0.5">
|
||||
<div className="flex w-full">
|
||||
<div className="w-0 grow truncate text-xs text-text-secondary">{fileItem.file.name}</div>
|
||||
</div>
|
||||
<div className="w-full truncate text-2xs/3 text-text-tertiary">
|
||||
<span className="uppercase">{getFileType(fileItem.file)}</span>
|
||||
<span className="px-1 text-text-quaternary">·</span>
|
||||
<span>{formatFileSize(fileItem.file.size)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex w-16 shrink-0 items-center justify-end gap-1 pr-3">
|
||||
{isUploading && (
|
||||
<SimplePieChart
|
||||
percentage={fileItem.progress}
|
||||
stroke={chartColor}
|
||||
fill={chartColor}
|
||||
animationDuration={0}
|
||||
/>
|
||||
)}
|
||||
{isError && <RiErrorWarningFill className="size-4 text-text-destructive" />}
|
||||
<span
|
||||
className="flex size-6 cursor-pointer items-center justify-center"
|
||||
onClick={handleRemove}
|
||||
>
|
||||
<RiDeleteBinLine className="size-4 text-text-tertiary" />
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default FileListItem
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
import type { ChangeEvent, RefObject } from 'react'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { useAtomValue } from 'jotai'
|
||||
import { useId } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { deploymentEditionAtom } from '@/features/system-features/state'
|
||||
|
||||
type FileUploadConfig = {
|
||||
file_size_limit: number
|
||||
batch_count_limit: number
|
||||
file_upload_limit: number
|
||||
}
|
||||
|
||||
export type UploadDropzoneProps = {
|
||||
dropRef: RefObject<HTMLDivElement | null>
|
||||
dragRef: RefObject<HTMLDivElement | null>
|
||||
fileUploaderRef: RefObject<HTMLInputElement | null>
|
||||
dragging: boolean
|
||||
supportBatchUpload: boolean
|
||||
supportTypesShowNames: string
|
||||
fileUploadConfig: FileUploadConfig
|
||||
acceptTypes: string[]
|
||||
onSelectFile: () => void
|
||||
onFileChange: (e: ChangeEvent<HTMLInputElement>) => void
|
||||
allowedExtensions: string[]
|
||||
}
|
||||
|
||||
const UploadDropzone = ({
|
||||
dropRef,
|
||||
dragRef,
|
||||
fileUploaderRef,
|
||||
dragging,
|
||||
supportBatchUpload,
|
||||
supportTypesShowNames,
|
||||
fileUploadConfig,
|
||||
acceptTypes,
|
||||
onSelectFile,
|
||||
onFileChange,
|
||||
allowedExtensions,
|
||||
}: UploadDropzoneProps) => {
|
||||
const { t } = useTranslation(['datasetCreation'])
|
||||
const browseLabelId = useId()
|
||||
const canBrowse = allowedExtensions.length > 0
|
||||
const deploymentEdition = useAtomValue(deploymentEditionAtom)
|
||||
|
||||
return (
|
||||
<>
|
||||
<input
|
||||
ref={fileUploaderRef}
|
||||
id="fileUploader"
|
||||
className="hidden"
|
||||
type="file"
|
||||
aria-labelledby={canBrowse ? browseLabelId : undefined}
|
||||
aria-label={
|
||||
canBrowse ? undefined : t(($) => $['stepOne.uploader.browse'], { ns: 'datasetCreation' })
|
||||
}
|
||||
multiple={supportBatchUpload}
|
||||
accept={acceptTypes.join(',')}
|
||||
onChange={onFileChange}
|
||||
/>
|
||||
<div
|
||||
ref={dropRef}
|
||||
className={cn(
|
||||
'relative box-border flex min-h-20 flex-col items-center justify-center gap-1 rounded-xl border border-dashed border-components-dropzone-border bg-components-dropzone-bg px-4 py-3 text-xs/4 text-text-tertiary',
|
||||
dragging && 'border-components-dropzone-border-accent bg-components-dropzone-bg-accent',
|
||||
)}
|
||||
>
|
||||
<div className="flex min-h-5 items-center justify-center text-sm/4 text-text-secondary">
|
||||
<span className="mr-2 i-ri-upload-cloud-2-line size-5" />
|
||||
<span>
|
||||
{supportBatchUpload
|
||||
? t(($) => $['stepOne.uploader.button'], { ns: 'datasetCreation' })
|
||||
: t(($) => $['stepOne.uploader.buttonSingleFile'], { ns: 'datasetCreation' })}
|
||||
{canBrowse && (
|
||||
<button
|
||||
id={browseLabelId}
|
||||
type="button"
|
||||
className="font-inherit ml-1 inline cursor-pointer appearance-none border-0 bg-transparent p-0 text-text-accent"
|
||||
onClick={onSelectFile}
|
||||
>
|
||||
{t(($) => $['stepOne.uploader.browse'], { ns: 'datasetCreation' })}
|
||||
</button>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
<div>
|
||||
{deploymentEdition === 'CLOUD'
|
||||
? t(($) => $['stepOne.uploader.tipWithTotalLimit'], {
|
||||
ns: 'datasetCreation',
|
||||
size: fileUploadConfig.file_size_limit,
|
||||
supportTypes: supportTypesShowNames,
|
||||
batchCount: fileUploadConfig.batch_count_limit,
|
||||
totalCount: fileUploadConfig.file_upload_limit,
|
||||
})
|
||||
: t(($) => $['stepOne.uploader.tip'], {
|
||||
ns: 'datasetCreation',
|
||||
size: fileUploadConfig.file_size_limit,
|
||||
supportTypes: supportTypesShowNames,
|
||||
batchCount: fileUploadConfig.batch_count_limit,
|
||||
})}
|
||||
</div>
|
||||
{dragging && <div ref={dragRef} className="absolute top-0 left-0 size-full" />}
|
||||
</div>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default UploadDropzone
|
||||
+1
@@ -0,0 +1 @@
|
||||
export const PROGRESS_ERROR = -2
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
import { act, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
|
||||
import { createFileUploadConfigFixture } from '@/test/fixtures/file-upload-config'
|
||||
import { createDataSourceStore } from '../../../store'
|
||||
import { DataSourceContext } from '../../../store/provider'
|
||||
import { useLocalFileUpload } from '../use-local-file-upload'
|
||||
|
||||
const upload = vi.fn()
|
||||
vi.mock('@/service/base', () => ({ upload: (...args: unknown[]) => upload(...args) }))
|
||||
vi.mock('@/app/notifications', () => ({ toast: { error: vi.fn() } }))
|
||||
|
||||
function Uploader() {
|
||||
const { fileUploaderRef, fileChangeHandle, localFileList, removeFile } = useLocalFileUpload({
|
||||
allowedExtensions: ['pdf'],
|
||||
})
|
||||
return (
|
||||
<>
|
||||
<input
|
||||
aria-label="Upload documents"
|
||||
type="file"
|
||||
ref={fileUploaderRef}
|
||||
onChange={fileChangeHandle}
|
||||
/>
|
||||
{localFileList.map((item) => (
|
||||
<button key={item.fileID} onClick={() => removeFile(item.fileID)}>
|
||||
Remove {item.file.name}
|
||||
</button>
|
||||
))}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
function setup() {
|
||||
const store = createDataSourceStore()
|
||||
const existing = {
|
||||
fileID: 'existing',
|
||||
file: Object.assign(new File(['existing'], 'Existing.pdf'), { id: 'server-existing' }),
|
||||
progress: 100,
|
||||
}
|
||||
store.getState().setLocalFileList([existing])
|
||||
const queryClient = createConsoleQueryClient()
|
||||
queryClient.setQueryData(
|
||||
consoleQuery.files.upload.get.queryOptions().queryKey,
|
||||
createFileUploadConfigFixture(),
|
||||
)
|
||||
queryClient.setQueryData(consoleQuery.files.supportType.get.queryOptions().queryKey, {
|
||||
allowed_extensions: ['pdf'],
|
||||
})
|
||||
renderWithConsoleQuery(
|
||||
<DataSourceContext value={store}>
|
||||
<Uploader />
|
||||
</DataSourceContext>,
|
||||
{ queryClient, systemFeatures: { deployment_edition: 'CLOUD' } },
|
||||
)
|
||||
return { store, existing }
|
||||
}
|
||||
|
||||
// Generic validation, drop handling and limits belong to useFileUpload. This
|
||||
// adapter owns merging upload transitions into the pipeline's data-source store.
|
||||
it('updates only the uploaded file and preserves other files through progress, success and removal', async () => {
|
||||
const user = userEvent.setup()
|
||||
let resolve!: (value: { id: string; name: string }) => void
|
||||
upload.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((res) => {
|
||||
resolve = res
|
||||
}),
|
||||
)
|
||||
const { store, existing } = setup()
|
||||
await user.upload(
|
||||
screen.getByLabelText('Upload documents'),
|
||||
new File(['new'], 'New.pdf', { type: 'application/pdf' }),
|
||||
)
|
||||
expect(store.getState().localFileList).toHaveLength(2)
|
||||
const request = upload.mock.lastCall![0]
|
||||
act(() => request.onprogress({ lengthComputable: true, loaded: 53, total: 100 }))
|
||||
expect(store.getState().localFileList[1]?.progress).toBe(53)
|
||||
expect(store.getState().localFileList[0]).toEqual(existing)
|
||||
await act(async () => resolve({ id: 'server-new', name: 'New.pdf' }))
|
||||
await waitFor(() =>
|
||||
expect(store.getState().localFileList[1]).toEqual({
|
||||
fileID: expect.any(String),
|
||||
file: { id: 'server-new', name: 'New.pdf' },
|
||||
progress: 100,
|
||||
}),
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: 'Remove New.pdf' }))
|
||||
expect(store.getState().localFileList).toEqual([existing])
|
||||
})
|
||||
|
||||
it('marks a failed upload without losing existing documents', async () => {
|
||||
const user = userEvent.setup()
|
||||
upload.mockRejectedValueOnce(new Error('offline'))
|
||||
const { store, existing } = setup()
|
||||
await user.upload(
|
||||
screen.getByLabelText('Upload documents'),
|
||||
new File(['new'], 'New.pdf', { type: 'application/pdf' }),
|
||||
)
|
||||
await waitFor(() => expect(store.getState().localFileList[1]?.progress).toBe(-2))
|
||||
expect(store.getState().localFileList[0]).toEqual(existing)
|
||||
expect(store.getState().localFileList[1]?.file.name).toBe('New.pdf')
|
||||
})
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import { produce } from 'immer'
|
||||
import { useCallback } from 'react'
|
||||
import { useFileUpload } from '@/app/components/datasets/create/file-uploader/hooks/use-file-upload'
|
||||
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../../store'
|
||||
|
||||
type UseLocalFileUploadOptions = {
|
||||
allowedExtensions: string[]
|
||||
supportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for handling local file uploads in the create-from-pipeline flow.
|
||||
* This is a thin wrapper around the generic useFileUpload hook that provides
|
||||
* Zustand store integration for state management.
|
||||
*/
|
||||
export const useLocalFileUpload = ({
|
||||
allowedExtensions,
|
||||
supportBatchUpload = true,
|
||||
}: UseLocalFileUploadOptions) => {
|
||||
const localFileList = useDataSourceStoreWithSelector((state) => state.localFileList)
|
||||
const dataSourceStore = useDataSourceStore()
|
||||
|
||||
const prepareFileList = useCallback(
|
||||
(files: FileItem[]) => {
|
||||
const { setLocalFileList } = dataSourceStore.getState()
|
||||
setLocalFileList(files)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const onFileUpdate = useCallback(
|
||||
(fileItem: FileItem, progress: number, list: FileItem[]) => {
|
||||
const { setLocalFileList } = dataSourceStore.getState()
|
||||
const newList = produce(list, (draft) => {
|
||||
const targetIndex = draft.findIndex((file) => file.fileID === fileItem.fileID)
|
||||
if (targetIndex !== -1) {
|
||||
draft[targetIndex] = {
|
||||
...draft[targetIndex],
|
||||
...fileItem,
|
||||
progress,
|
||||
}
|
||||
}
|
||||
})
|
||||
setLocalFileList(newList)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const onFileListUpdate = useCallback(
|
||||
(files: FileItem[]) => {
|
||||
const { setLocalFileList } = dataSourceStore.getState()
|
||||
setLocalFileList(files)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const onPreview = useCallback(
|
||||
(file: File) => {
|
||||
const { setCurrentLocalFile } = dataSourceStore.getState()
|
||||
setCurrentLocalFile(file)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const {
|
||||
dropRef,
|
||||
dragRef,
|
||||
fileUploaderRef,
|
||||
dragging,
|
||||
fileUploadConfig,
|
||||
acceptTypes,
|
||||
supportTypesShowNames,
|
||||
hideUpload,
|
||||
selectHandle,
|
||||
fileChangeHandle,
|
||||
removeFile,
|
||||
handlePreview,
|
||||
} = useFileUpload({
|
||||
fileList: localFileList,
|
||||
prepareFileList,
|
||||
onFileUpdate,
|
||||
onFileListUpdate,
|
||||
onPreview,
|
||||
supportBatchUpload,
|
||||
allowedExtensions,
|
||||
})
|
||||
|
||||
return {
|
||||
// Refs
|
||||
dropRef,
|
||||
dragRef,
|
||||
fileUploaderRef,
|
||||
|
||||
// State
|
||||
dragging,
|
||||
localFileList,
|
||||
|
||||
// Config
|
||||
fileUploadConfig,
|
||||
acceptTypes,
|
||||
supportTypesShowNames,
|
||||
hideUpload,
|
||||
|
||||
// Handlers
|
||||
selectHandle,
|
||||
fileChangeHandle,
|
||||
removeFile,
|
||||
handlePreview,
|
||||
}
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
'use client'
|
||||
import FileListItem from './components/file-list-item'
|
||||
import UploadDropzone from './components/upload-dropzone'
|
||||
import { useLocalFileUpload } from './hooks/use-local-file-upload'
|
||||
|
||||
type LocalFileProps = {
|
||||
allowedExtensions: string[]
|
||||
supportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const LocalFile = ({ allowedExtensions, supportBatchUpload = true }: LocalFileProps) => {
|
||||
const {
|
||||
dropRef,
|
||||
dragRef,
|
||||
fileUploaderRef,
|
||||
dragging,
|
||||
localFileList,
|
||||
fileUploadConfig,
|
||||
acceptTypes,
|
||||
supportTypesShowNames,
|
||||
hideUpload,
|
||||
selectHandle,
|
||||
fileChangeHandle,
|
||||
removeFile,
|
||||
handlePreview,
|
||||
} = useLocalFileUpload({ allowedExtensions, supportBatchUpload })
|
||||
|
||||
return (
|
||||
<div className="flex flex-col">
|
||||
{!hideUpload && (
|
||||
<UploadDropzone
|
||||
dropRef={dropRef}
|
||||
dragRef={dragRef}
|
||||
fileUploaderRef={fileUploaderRef}
|
||||
dragging={dragging}
|
||||
supportBatchUpload={supportBatchUpload}
|
||||
supportTypesShowNames={supportTypesShowNames}
|
||||
fileUploadConfig={fileUploadConfig}
|
||||
acceptTypes={acceptTypes}
|
||||
onSelectFile={selectHandle}
|
||||
onFileChange={fileChangeHandle}
|
||||
allowedExtensions={allowedExtensions}
|
||||
/>
|
||||
)}
|
||||
{localFileList.length > 0 && (
|
||||
<div className="mt-1 flex flex-col gap-y-1">
|
||||
{localFileList.map((fileItem, index) => (
|
||||
<FileListItem
|
||||
key={`${fileItem.fileID}-${index}`}
|
||||
fileItem={fileItem}
|
||||
onPreview={handlePreview}
|
||||
onRemove={removeFile}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default LocalFile
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import type { DataSourceNotionWorkspace } from '@/models/common'
|
||||
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
|
||||
import { act, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
|
||||
import { BlockEnum } from '@/app/components/workflow/types'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
|
||||
import { createNuqsTestRenderer } from '@/test/nuqs-testing'
|
||||
import { createDataSourceStore } from '../../store'
|
||||
import { DataSourceContext } from '../../store/provider'
|
||||
import OnlineDocuments from '../index'
|
||||
|
||||
const { ssePost, toastError } = vi.hoisted(() => ({ ssePost: vi.fn(), toastError: vi.fn() }))
|
||||
vi.mock('@tanstack/react-virtual')
|
||||
vi.mock('@/service/base', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('@/service/base')>()),
|
||||
ssePost,
|
||||
}))
|
||||
vi.mock('@/app/notifications', () => ({ toast: { error: toastError } }))
|
||||
vi.mock('@/context/dataset-detail', () => ({
|
||||
useDatasetDetailContextWithSelector: (
|
||||
select: (state: { dataset: { pipeline_id: string } }) => unknown,
|
||||
) => select({ dataset: { pipeline_id: 'pipeline-123' } }),
|
||||
}))
|
||||
const onUrlUpdate = vi.fn()
|
||||
const render = createNuqsTestRenderer(renderWithConsoleQuery, () => ({ onUrlUpdate }))
|
||||
const nodeData: DataSourceNodeType = {
|
||||
type: BlockEnum.DataSource,
|
||||
title: 'Notion',
|
||||
desc: '',
|
||||
provider_type: DatasourceType.onlineDocument,
|
||||
plugin_id: 'plugin-123',
|
||||
provider_name: 'notion',
|
||||
datasource_name: 'notion',
|
||||
datasource_label: 'Notion',
|
||||
datasource_parameters: { workspace: { type: VarKindType.constant, value: 'workspace-1' } },
|
||||
datasource_configurations: {},
|
||||
}
|
||||
const documents: DataSourceNotionWorkspace[] = [
|
||||
{
|
||||
workspace_name: 'Workspace',
|
||||
workspace_id: 'workspace-1',
|
||||
workspace_icon: null,
|
||||
pages: [
|
||||
{
|
||||
page_id: 'report',
|
||||
page_name: 'Report',
|
||||
parent_id: 'root',
|
||||
type: 'page',
|
||||
page_icon: null,
|
||||
is_bound: false,
|
||||
},
|
||||
{
|
||||
page_id: 'notes',
|
||||
page_name: 'Notes',
|
||||
parent_id: 'root',
|
||||
type: 'page',
|
||||
page_icon: null,
|
||||
is_bound: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
]
|
||||
const credentials: DatasourceCredentialResponse[] = [
|
||||
{
|
||||
id: 'credential-1',
|
||||
name: 'First credential',
|
||||
avatar_url: '',
|
||||
type: 'oauth2',
|
||||
is_default: false,
|
||||
credential: {},
|
||||
},
|
||||
{
|
||||
id: 'credential-2',
|
||||
name: 'Second credential',
|
||||
avatar_url: '',
|
||||
type: 'oauth2',
|
||||
is_default: false,
|
||||
credential: {},
|
||||
},
|
||||
]
|
||||
function setup({ cached = true, credential = 'credential-1', single = false, draft = false } = {}) {
|
||||
const store = createDataSourceStore()
|
||||
store.setState({ documentsData: cached ? documents : undefined, currentCredentialId: credential })
|
||||
const queryClient = createConsoleQueryClient()
|
||||
queryClient.setQueryData(
|
||||
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
|
||||
input: { params: { provider_id: 'plugin-123/notion' } },
|
||||
}).queryKey,
|
||||
{ result: credential ? credentials : [] },
|
||||
)
|
||||
const onCredentialChange = vi.fn((id: string) => store.getState().setCurrentCredentialId(id))
|
||||
render(
|
||||
<DataSourceContext value={store}>
|
||||
<OnlineDocuments
|
||||
nodeId="node-1"
|
||||
nodeData={nodeData}
|
||||
onCredentialChange={onCredentialChange}
|
||||
supportBatchUpload={!single}
|
||||
isInPipeline={draft}
|
||||
/>
|
||||
</DataSourceContext>,
|
||||
{ queryClient },
|
||||
)
|
||||
return { store, onCredentialChange }
|
||||
}
|
||||
function callbacks(): {
|
||||
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => void
|
||||
onDataSourceNodeError: (data: DataSourceNodeErrorResponse) => void
|
||||
} {
|
||||
return ssePost.mock.lastCall![2]
|
||||
}
|
||||
beforeEach(() => vi.clearAllMocks())
|
||||
|
||||
it.each([true, false])(
|
||||
'loads documents through the draft=%s route using resolved node parameters',
|
||||
async (draft) => {
|
||||
const { store } = setup({ cached: false, draft })
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
expect(ssePost).toHaveBeenCalledWith(
|
||||
`/rag/pipelines/pipeline-123/workflows/${draft ? 'draft' : 'published'}/datasource/nodes/node-1/run`,
|
||||
{
|
||||
body: {
|
||||
inputs: { workspace: 'workspace-1' },
|
||||
credential_id: 'credential-1',
|
||||
datasource_type: DatasourceType.onlineDocument,
|
||||
},
|
||||
},
|
||||
expect.any(Object),
|
||||
)
|
||||
act(() =>
|
||||
callbacks().onDataSourceNodeCompleted({
|
||||
event: 'datasource_completed',
|
||||
time_consuming: 1,
|
||||
data: documents,
|
||||
}),
|
||||
)
|
||||
expect(await screen.findByRole('checkbox', { name: 'Report' })).toBeInTheDocument()
|
||||
expect(store.getState().documentsData).toEqual(documents)
|
||||
expect(screen.queryByTestId('notion-page-preview-report') !== null).toBe(!draft)
|
||||
},
|
||||
)
|
||||
|
||||
it('waits for a credential before requesting documents', () => {
|
||||
setup({ cached: false, credential: '' })
|
||||
expect(ssePost).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('selects pages with their workspace and removes only the deselected page', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { store } = setup()
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Report' }))
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Notes' }))
|
||||
expect(store.getState().selectedPagesId).toEqual(new Set(['report', 'notes']))
|
||||
expect(store.getState().onlineDocuments).toEqual(
|
||||
documents[0]!.pages.map((page) => ({ ...page, workspace_id: 'workspace-1' })),
|
||||
)
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Report' }))
|
||||
expect(store.getState().onlineDocuments).toEqual([
|
||||
{ ...documents[0]!.pages[1], workspace_id: 'workspace-1' },
|
||||
])
|
||||
expect(screen.getByRole('checkbox', { name: 'Notes' })).toBeChecked()
|
||||
})
|
||||
|
||||
it('replaces the selected page in single-upload mode', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { store } = setup({ single: true })
|
||||
await user.click(screen.getByRole('radio', { name: 'Report' }))
|
||||
await user.click(screen.getByRole('radio', { name: 'Notes' }))
|
||||
expect(store.getState().selectedPagesId).toEqual(new Set(['notes']))
|
||||
expect(store.getState().onlineDocuments).toEqual([
|
||||
{ ...documents[0]!.pages[1], workspace_id: 'workspace-1' },
|
||||
])
|
||||
expect(screen.getByRole('radio', { name: 'Report' })).not.toBeChecked()
|
||||
})
|
||||
|
||||
it('filters documents without losing selection and previews the chosen page', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { store } = setup()
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Report' }))
|
||||
await user.type(
|
||||
screen.getByPlaceholderText('common.dataSource.notion.selector.searchPages'),
|
||||
'Notes',
|
||||
)
|
||||
await waitFor(() =>
|
||||
expect(screen.queryByRole('checkbox', { name: 'Report' })).not.toBeInTheDocument(),
|
||||
)
|
||||
await user.click(screen.getByTestId('notion-page-preview-notes'))
|
||||
expect(store.getState().currentDocument).toEqual({
|
||||
...documents[0]!.pages[1],
|
||||
workspace_id: 'workspace-1',
|
||||
})
|
||||
expect(store.getState().selectedPagesId).toEqual(new Set(['report']))
|
||||
await user.click(screen.getByRole('button', { name: 'common.operation.clear' }))
|
||||
expect(await screen.findByRole('checkbox', { name: 'Report' })).toBeChecked()
|
||||
})
|
||||
|
||||
it('requests documents again with the credential selected in the header', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { onCredentialChange } = setup()
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
await user.click(screen.getByRole('button', { name: /First credential/ }))
|
||||
await user.click(await screen.findByText('Second credential'))
|
||||
expect(onCredentialChange).toHaveBeenCalledExactlyOnceWith('credential-2')
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledTimes(2))
|
||||
expect(ssePost.mock.lastCall![1].body.credential_id).toBe('credential-2')
|
||||
})
|
||||
|
||||
it('reports stream failures and opens data-source configuration', async () => {
|
||||
const user = userEvent.setup()
|
||||
setup({ cached: false })
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
act(() =>
|
||||
callbacks().onDataSourceNodeError({
|
||||
event: 'datasource_error',
|
||||
error: 'Notion request failed',
|
||||
}),
|
||||
)
|
||||
expect(toastError).toHaveBeenCalledExactlyOnceWith('Notion request failed')
|
||||
await user.click(screen.getByRole('button', { name: /^datasetPipeline\.configurationTip/ }))
|
||||
await waitFor(() =>
|
||||
expect(onUrlUpdate.mock.lastCall![0].searchParams.get('settings')).toBe('data-source'),
|
||||
)
|
||||
})
|
||||
+195
@@ -0,0 +1,195 @@
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
|
||||
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { useQueryState } from 'nuqs'
|
||||
import { useCallback, useEffect, useMemo } from 'react'
|
||||
import { useShallow } from 'zustand/react/shallow'
|
||||
import { LoadingPlaceholder } from '@/app/components/base/loading-placeholder'
|
||||
import SearchInput from '@/app/components/base/notion-page-selector/search-input'
|
||||
import {
|
||||
settingsQueryParamName,
|
||||
settingsQueryParser,
|
||||
} from '@/app/components/header/account-setting/query-params'
|
||||
import { toast } from '@/app/notifications'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { useDocLink } from '@/context/i18n'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import Header from '../base/header'
|
||||
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
|
||||
import PageSelector from './page-selector'
|
||||
import Title from './title'
|
||||
|
||||
type OnlineDocumentsProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
onCredentialChange: (credentialId: string) => void
|
||||
isInPipeline?: boolean
|
||||
supportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const OnlineDocuments = ({
|
||||
nodeId,
|
||||
nodeData,
|
||||
isInPipeline = false,
|
||||
supportBatchUpload = true,
|
||||
onCredentialChange,
|
||||
}: OnlineDocumentsProps) => {
|
||||
const docLink = useDocLink()
|
||||
const pipelineId = useDatasetDetailContextWithSelector((s) => s.dataset?.pipeline_id)
|
||||
const [, setSettingsDestination] = useQueryState(settingsQueryParamName, settingsQueryParser)
|
||||
const { documentsData, searchValue, selectedPagesId, currentCredentialId } =
|
||||
useDataSourceStoreWithSelector(
|
||||
useShallow((state) => ({
|
||||
documentsData: state.documentsData,
|
||||
searchValue: state.searchValue,
|
||||
selectedPagesId: state.selectedPagesId,
|
||||
currentCredentialId: state.currentCredentialId,
|
||||
})),
|
||||
)
|
||||
|
||||
const { data: dataSourceAuth } = useQuery(
|
||||
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
|
||||
input: { params: { provider_id: `${nodeData.plugin_id}/${nodeData.provider_name}` } },
|
||||
}),
|
||||
)
|
||||
|
||||
const dataSourceStore = useDataSourceStore()
|
||||
|
||||
const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
|
||||
const pagesMap = (documentsData || []).reduce(
|
||||
(prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
|
||||
next.pages.forEach((page) => {
|
||||
prev[page.page_id] = {
|
||||
...page,
|
||||
workspace_id: next.workspace_id,
|
||||
}
|
||||
})
|
||||
|
||||
return prev
|
||||
},
|
||||
{},
|
||||
)
|
||||
return pagesMap
|
||||
}, [documentsData])
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const getOnlineDocuments = useCallback(async () => {
|
||||
const { currentCredentialId } = dataSourceStore.getState()
|
||||
// Convert datasource_parameters to inputs format for the API
|
||||
const inputs = Object.entries(nodeData.datasource_parameters || {}).reduce(
|
||||
(acc, [key, value]) => {
|
||||
acc[key] =
|
||||
typeof value === 'object' && value !== null && 'value' in value ? value.value : value
|
||||
return acc
|
||||
},
|
||||
{} as Record<string, any>,
|
||||
)
|
||||
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs,
|
||||
credential_id: currentCredentialId,
|
||||
datasource_type: DatasourceType.onlineDocument,
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
|
||||
const { setDocumentsData } = dataSourceStore.getState()
|
||||
setDocumentsData(documentsData.data as DataSourceNotionWorkspace[])
|
||||
},
|
||||
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
|
||||
toast.error(error.error)
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [dataSourceStore, datasourceNodeRunURL, nodeData.datasource_parameters])
|
||||
|
||||
useEffect(() => {
|
||||
if (!currentCredentialId) return
|
||||
getOnlineDocuments()
|
||||
}, [currentCredentialId, getOnlineDocuments])
|
||||
|
||||
const handleSearchValueChange = useCallback(
|
||||
(value: string) => {
|
||||
const { setSearchValue } = dataSourceStore.getState()
|
||||
setSearchValue(value)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const handleSelectPages = useCallback(
|
||||
(newSelectedPagesId: Set<string>) => {
|
||||
const { setSelectedPagesId, setOnlineDocuments } = dataSourceStore.getState()
|
||||
const selectedPages = Array.from(newSelectedPagesId).map(
|
||||
(pageId) => PagesMapAndSelectedPagesId[pageId]!,
|
||||
)
|
||||
setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
|
||||
setOnlineDocuments(selectedPages)
|
||||
},
|
||||
[dataSourceStore, PagesMapAndSelectedPagesId],
|
||||
)
|
||||
|
||||
const handlePreviewPage = useCallback(
|
||||
(previewPageId: string) => {
|
||||
const { setCurrentDocument } = dataSourceStore.getState()
|
||||
setCurrentDocument(PagesMapAndSelectedPagesId[previewPageId])
|
||||
},
|
||||
[PagesMapAndSelectedPagesId, dataSourceStore],
|
||||
)
|
||||
|
||||
const handleSetting = useCallback(() => {
|
||||
setSettingsDestination('data-source')
|
||||
}, [setSettingsDestination])
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-2">
|
||||
<Header
|
||||
docTitle="Docs"
|
||||
docLink={docLink('/use-dify/knowledge/knowledge-pipeline/authorize-data-source')}
|
||||
onClickConfiguration={handleSetting}
|
||||
pluginName={nodeData.datasource_label}
|
||||
currentCredentialId={currentCredentialId}
|
||||
onCredentialChange={onCredentialChange}
|
||||
credentials={dataSourceAuth?.result || []}
|
||||
/>
|
||||
<div className="rounded-xl border border-components-panel-border bg-background-default-subtle">
|
||||
<div className="flex items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-1 pl-3">
|
||||
<div className="flex grow items-center">
|
||||
<Title name={nodeData.datasource_label} />
|
||||
</div>
|
||||
<SearchInput value={searchValue} onChange={handleSearchValueChange} />
|
||||
</div>
|
||||
<div className="overflow-hidden rounded-b-xl">
|
||||
{documentsData?.length ? (
|
||||
<PageSelector
|
||||
key={`${currentCredentialId}:${supportBatchUpload ? 'multiple' : 'single'}`}
|
||||
checkedIds={selectedPagesId}
|
||||
disabledValue={new Set()}
|
||||
searchValue={searchValue}
|
||||
list={documentsData[0]!.pages || []}
|
||||
pagesMap={PagesMapAndSelectedPagesId}
|
||||
onSelect={handleSelectPages}
|
||||
canPreview={!isInPipeline}
|
||||
onPreview={handlePreviewPage}
|
||||
isMultipleChoice={supportBatchUpload}
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-74 items-center justify-center">
|
||||
<LoadingPlaceholder className="h-full" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default OnlineDocuments
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
import type { ComponentProps } from 'react'
|
||||
import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
|
||||
import { render, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { useState } from 'react'
|
||||
import PageSelector from '../index'
|
||||
|
||||
vi.mock('@tanstack/react-virtual')
|
||||
const page = (id: string, name: string, parent = 'root'): DataSourceNotionPage => ({
|
||||
page_id: id,
|
||||
page_name: name,
|
||||
parent_id: parent,
|
||||
page_icon: null,
|
||||
is_bound: false,
|
||||
type: 'page',
|
||||
})
|
||||
const pages = [
|
||||
page('parent', 'Parent'),
|
||||
page('child', 'Child', 'parent'),
|
||||
page('grandchild', 'Grandchild', 'child'),
|
||||
page('other', 'Other'),
|
||||
]
|
||||
const mapPages = (list: DataSourceNotionPage[]): DataSourceNotionPageMap =>
|
||||
Object.fromEntries(list.map((item) => [item.page_id, { ...item, workspace_id: 'workspace' }]))
|
||||
const defaults = {
|
||||
list: pages,
|
||||
pagesMap: mapPages(pages),
|
||||
disabledValue: new Set<string>(),
|
||||
searchValue: '',
|
||||
}
|
||||
function Harness(props: Partial<ComponentProps<typeof PageSelector>>) {
|
||||
const [selected, setSelected] = useState(props.checkedIds ?? new Set<string>())
|
||||
return (
|
||||
<PageSelector
|
||||
{...defaults}
|
||||
{...props}
|
||||
checkedIds={selected}
|
||||
onSelect={(ids) => {
|
||||
setSelected(ids)
|
||||
props.onSelect?.(ids)
|
||||
}}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
it('expands nested pages and collapses descendants when their parent is collapsed', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(<Harness />)
|
||||
expect(screen.queryByRole('checkbox', { name: 'Child' })).not.toBeInTheDocument()
|
||||
await user.click(screen.getByTestId('notion-page-toggle-parent'))
|
||||
await user.click(screen.getByTestId('notion-page-toggle-child'))
|
||||
expect(screen.getByRole('checkbox', { name: 'Grandchild' })).toBeInTheDocument()
|
||||
await user.click(screen.getByTestId('notion-page-toggle-parent'))
|
||||
expect(screen.queryByRole('checkbox', { name: 'Child' })).not.toBeInTheDocument()
|
||||
expect(screen.queryByRole('checkbox', { name: 'Grandchild' })).not.toBeInTheDocument()
|
||||
await user.click(screen.getByTestId('notion-page-toggle-parent'))
|
||||
expect(screen.getByRole('checkbox', { name: 'Child' })).toBeInTheDocument()
|
||||
expect(screen.queryByRole('checkbox', { name: 'Grandchild' })).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('selects and deselects a complete subtree without changing an independent selection', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onSelect = vi.fn()
|
||||
render(<Harness checkedIds={new Set(['other'])} onSelect={onSelect} />)
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Parent' }))
|
||||
expect(onSelect).toHaveBeenLastCalledWith(new Set(['other', 'parent', 'child', 'grandchild']))
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Parent' }))
|
||||
expect(onSelect).toHaveBeenLastCalledWith(new Set(['other']))
|
||||
expect(screen.getByRole('checkbox', { name: 'Other' })).toBeChecked()
|
||||
})
|
||||
|
||||
it('selects only the matched page in search mode and shows its ancestors', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onSelect = vi.fn()
|
||||
render(<Harness searchValue="Child" onSelect={onSelect} />)
|
||||
expect(screen.queryByRole('checkbox', { name: 'Parent' })).not.toBeInTheDocument()
|
||||
expect(screen.getByTitle('Parent / Child')).toBeInTheDocument()
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Child' }))
|
||||
expect(onSelect).toHaveBeenCalledExactlyOnceWith(new Set(['child']))
|
||||
})
|
||||
|
||||
it('selects one page without cascading in single-choice mode', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onSelect = vi.fn()
|
||||
render(<Harness isMultipleChoice={false} onSelect={onSelect} />)
|
||||
await user.click(screen.getByRole('radio', { name: 'Parent' }))
|
||||
expect(onSelect).toHaveBeenLastCalledWith(new Set(['parent']))
|
||||
await user.click(screen.getByRole('radio', { name: 'Other' }))
|
||||
expect(onSelect).toHaveBeenLastCalledWith(new Set(['other']))
|
||||
expect(screen.getByRole('radio', { name: 'Parent' })).not.toBeChecked()
|
||||
})
|
||||
|
||||
it.each([true, false])('blocks a disabled page in multiple-choice=%s mode', async (multiple) => {
|
||||
const user = userEvent.setup()
|
||||
const onSelect = vi.fn()
|
||||
render(
|
||||
<Harness isMultipleChoice={multiple} disabledValue={new Set(['other'])} onSelect={onSelect} />,
|
||||
)
|
||||
await user.click(screen.getByRole(multiple ? 'checkbox' : 'radio', { name: 'Other' }))
|
||||
expect(onSelect).not.toHaveBeenCalled()
|
||||
await user.click(screen.getByRole(multiple ? 'checkbox' : 'radio', { name: 'Parent' }))
|
||||
expect(onSelect).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('previews the requested page without selecting it and hides preview when disabled', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onSelect = vi.fn()
|
||||
const onPreview = vi.fn()
|
||||
const { rerender } = render(<Harness onSelect={onSelect} onPreview={onPreview} />)
|
||||
await user.click(screen.getByTestId('notion-page-preview-other'))
|
||||
expect(onPreview).toHaveBeenCalledExactlyOnceWith('other')
|
||||
expect(onSelect).not.toHaveBeenCalled()
|
||||
rerender(<Harness canPreview={false} onSelect={onSelect} onPreview={onPreview} />)
|
||||
expect(screen.queryByTestId('notion-page-preview-other')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('updates visible results when documents or the search change', async () => {
|
||||
const { rerender } = render(<Harness list={[]} pagesMap={{}} />)
|
||||
expect(screen.getByText('common.dataSource.notion.selector.noSearchResult')).toBeInTheDocument()
|
||||
const list = [page('orphan', 'Orphan', 'missing-parent')]
|
||||
rerender(<Harness list={list} pagesMap={mapPages(list)} />)
|
||||
expect(screen.getByRole('checkbox', { name: 'Orphan' })).toBeInTheDocument()
|
||||
rerender(<Harness list={list} pagesMap={mapPages(list)} searchValue="missing" />)
|
||||
await waitFor(() => expect(screen.queryByRole('checkbox')).not.toBeInTheDocument())
|
||||
expect(screen.getByText('common.dataSource.notion.selector.noSearchResult')).toBeInTheDocument()
|
||||
})
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { usePageSelectorModel } from '@/app/components/base/notion-page-selector/page-selector/use-page-selector-model'
|
||||
import VirtualPageList from '@/app/components/base/notion-page-selector/page-selector/virtual-page-list'
|
||||
|
||||
type PageSelectorProps = {
|
||||
checkedIds: Set<string>
|
||||
disabledValue: Set<string>
|
||||
searchValue: string
|
||||
pagesMap: DataSourceNotionPageMap
|
||||
list: DataSourceNotionPage[]
|
||||
onSelect: (selectedPagesId: Set<string>) => void
|
||||
canPreview?: boolean
|
||||
onPreview?: (selectedPageId: string) => void
|
||||
isMultipleChoice?: boolean
|
||||
currentCredentialId?: string
|
||||
}
|
||||
|
||||
const PageSelector = ({
|
||||
checkedIds,
|
||||
disabledValue,
|
||||
searchValue,
|
||||
pagesMap,
|
||||
list,
|
||||
onSelect,
|
||||
canPreview = true,
|
||||
onPreview,
|
||||
isMultipleChoice = true,
|
||||
currentCredentialId: _currentCredentialId,
|
||||
}: PageSelectorProps) => {
|
||||
const { t } = useTranslation(['common'])
|
||||
const selectionMode = isMultipleChoice ? 'multiple' : 'single'
|
||||
const {
|
||||
currentPreviewPageId,
|
||||
effectiveSearchValue,
|
||||
rows,
|
||||
handlePreview,
|
||||
handleSelect,
|
||||
handleToggle,
|
||||
} = usePageSelectorModel({
|
||||
checkedIds,
|
||||
list,
|
||||
onPreview,
|
||||
onSelect,
|
||||
pagesMap,
|
||||
searchValue,
|
||||
selectionMode,
|
||||
})
|
||||
|
||||
if (!rows.length) {
|
||||
return (
|
||||
<div className="flex h-74 items-center justify-center text-[13px] text-text-tertiary">
|
||||
{t(($) => $['dataSource.notion.selector.noSearchResult'], { ns: 'common' })}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<VirtualPageList
|
||||
checkedIds={checkedIds}
|
||||
disabledValue={disabledValue}
|
||||
onPreview={handlePreview}
|
||||
onSelect={handleSelect}
|
||||
onToggle={handleToggle}
|
||||
previewPageId={currentPreviewPageId}
|
||||
rows={rows}
|
||||
searchValue={effectiveSearchValue}
|
||||
selectionMode={selectionMode}
|
||||
showPreview={canPreview}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
export default PageSelector
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type TitleProps = {
|
||||
name: string
|
||||
}
|
||||
|
||||
const Title = ({ name }: TitleProps) => {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
|
||||
return (
|
||||
<div className="px-1.25 py-1 system-sm-medium text-text-secondary">
|
||||
{t(($) => $['onlineDocument.pageSelectorTitle'], { ns: 'datasetPipeline', name })}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Title)
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
|
||||
import { act, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { BlockEnum } from '@/app/components/workflow/types'
|
||||
import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
|
||||
import { createNuqsTestRenderer } from '@/test/nuqs-testing'
|
||||
import { createDataSourceStore } from '../../store'
|
||||
import { DataSourceContext } from '../../store/provider'
|
||||
import OnlineDrive from '../index'
|
||||
|
||||
const { ssePost, toastError } = vi.hoisted(() => ({ ssePost: vi.fn(), toastError: vi.fn() }))
|
||||
vi.mock('@/service/base', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('@/service/base')>()),
|
||||
ssePost,
|
||||
}))
|
||||
vi.mock('@/app/notifications', () => ({ toast: { error: toastError } }))
|
||||
vi.mock('@/context/dataset-detail', () => ({
|
||||
useDatasetDetailContextWithSelector: (
|
||||
select: (state: { dataset: { pipeline_id: string } }) => unknown,
|
||||
) => select({ dataset: { pipeline_id: 'pipeline-123' } }),
|
||||
}))
|
||||
const onUrlUpdate = vi.fn()
|
||||
const render = createNuqsTestRenderer(renderWithConsoleQuery, () => ({ onUrlUpdate }))
|
||||
const nodeData: DataSourceNodeType = {
|
||||
type: BlockEnum.DataSource,
|
||||
title: 'Drive',
|
||||
desc: '',
|
||||
provider_type: DatasourceType.onlineDrive,
|
||||
plugin_id: 'plugin-123',
|
||||
provider_name: 'drive',
|
||||
datasource_name: 'drive',
|
||||
datasource_label: 'Drive',
|
||||
datasource_parameters: {},
|
||||
datasource_configurations: {},
|
||||
}
|
||||
const files: OnlineDriveFile[] = [
|
||||
{ id: 'file-1', name: 'Report.pdf', type: OnlineDriveFileType.file, size: 1024 },
|
||||
{ id: 'file-2', name: 'Other.txt', type: OnlineDriveFileType.file, size: 500 },
|
||||
{ id: 'folder-1', name: 'Archive', type: OnlineDriveFileType.folder },
|
||||
]
|
||||
function setup({ cached = true, credential = 'credential-1', single = false, draft = false } = {}) {
|
||||
const store = createDataSourceStore()
|
||||
store.setState({
|
||||
localFileList: [],
|
||||
onlineDriveFileList: cached ? files : [],
|
||||
currentCredentialId: credential,
|
||||
breadcrumbs: ['Root'],
|
||||
prefix: ['root'],
|
||||
bucket: 'bucket-a',
|
||||
nextPageParameters: { cursor: 'next' },
|
||||
})
|
||||
const queryClient = createConsoleQueryClient()
|
||||
queryClient.setQueryData(
|
||||
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
|
||||
input: { params: { provider_id: 'plugin-123/drive' } },
|
||||
}).queryKey,
|
||||
{ result: [] },
|
||||
)
|
||||
render(
|
||||
<DataSourceContext value={store}>
|
||||
<OnlineDrive
|
||||
nodeId="node-1"
|
||||
nodeData={nodeData}
|
||||
onCredentialChange={vi.fn()}
|
||||
supportBatchUpload={!single}
|
||||
isInPipeline={draft}
|
||||
/>
|
||||
</DataSourceContext>,
|
||||
{ queryClient },
|
||||
)
|
||||
return store
|
||||
}
|
||||
function callbacks(): {
|
||||
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => void
|
||||
onDataSourceNodeError: (data: DataSourceNodeErrorResponse) => void
|
||||
} {
|
||||
return ssePost.mock.lastCall![2]
|
||||
}
|
||||
beforeEach(() => vi.clearAllMocks())
|
||||
|
||||
it.each([true, false])(
|
||||
'loads the %s draft route and stores streamed files and pagination',
|
||||
async (draft) => {
|
||||
const store = setup({ cached: false, draft })
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
expect(ssePost).toHaveBeenCalledWith(
|
||||
`/rag/pipelines/pipeline-123/workflows/${draft ? 'draft' : 'published'}/datasource/nodes/node-1/run`,
|
||||
{
|
||||
body: {
|
||||
inputs: {
|
||||
prefix: 'root',
|
||||
bucket: 'bucket-a',
|
||||
next_page_parameters: { cursor: 'next' },
|
||||
max_keys: 30,
|
||||
},
|
||||
datasource_type: DatasourceType.onlineDrive,
|
||||
credential_id: 'credential-1',
|
||||
},
|
||||
},
|
||||
expect.any(Object),
|
||||
)
|
||||
act(() =>
|
||||
callbacks().onDataSourceNodeCompleted({
|
||||
event: 'datasource_completed',
|
||||
time_consuming: 1,
|
||||
data: [
|
||||
{
|
||||
bucket: 'bucket-a',
|
||||
files: [{ id: 'new-file', name: 'New.pdf', size: 1024, type: 'file' }],
|
||||
is_truncated: true,
|
||||
next_page_parameters: { cursor: 'next-page' },
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
expect(await screen.findByText('New.pdf')).toBeInTheDocument()
|
||||
expect(store.getState().isTruncated.current).toBe(true)
|
||||
expect(store.getState().currentNextPageParametersRef.current).toEqual({ cursor: 'next-page' })
|
||||
expect(store.getState().hasBucket).toBe(true)
|
||||
},
|
||||
)
|
||||
|
||||
it('does not request files without a credential', () => {
|
||||
setup({ cached: false, credential: '' })
|
||||
expect(ssePost).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uses cached files, selects them independently and clears selection when opening a folder', async () => {
|
||||
const user = userEvent.setup()
|
||||
const store = setup()
|
||||
expect(ssePost).not.toHaveBeenCalled()
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Report.pdf' }))
|
||||
expect(store.getState().selectedFileIds).toEqual(['file-1'])
|
||||
expect(screen.getByRole('checkbox', { name: 'Other.txt' })).not.toBeChecked()
|
||||
await user.click(screen.getByText('Archive'))
|
||||
expect(store.getState().selectedFileIds).toEqual([])
|
||||
expect(store.getState().breadcrumbs).toEqual(['Root', 'Archive'])
|
||||
expect(store.getState().prefix).toEqual(['root', 'folder-1'])
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
expect(ssePost.mock.lastCall![1].body.inputs.prefix).toBe('folder-1')
|
||||
})
|
||||
|
||||
it('keeps a single selected file when a second file is clicked in single-upload mode', async () => {
|
||||
const user = userEvent.setup()
|
||||
const store = setup({ single: true })
|
||||
await user.click(screen.getByText('Report.pdf'))
|
||||
await user.click(screen.getByText('Other.txt'))
|
||||
expect(store.getState().selectedFileIds).toEqual(['file-1'])
|
||||
})
|
||||
|
||||
it('reports a failed stream and opens datasource configuration', async () => {
|
||||
const user = userEvent.setup()
|
||||
setup({ cached: false })
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
act(() => callbacks().onDataSourceNodeError({ event: 'datasource_error', error: 'Drive failed' }))
|
||||
expect(toastError).toHaveBeenCalledWith('Drive failed')
|
||||
await user.click(screen.getByRole('button', { name: /^datasetPipeline\.configurationTip/ }))
|
||||
await waitFor(() =>
|
||||
expect(onUrlUpdate.mock.lastCall![0].searchParams.get('settings')).toBe('data-source'),
|
||||
)
|
||||
})
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
import type { OnlineDriveData } from '@/types/pipeline'
|
||||
import { describe, expect, it } from 'vite-plus/test'
|
||||
import { OnlineDriveFileType } from '@/models/pipeline'
|
||||
import { convertOnlineDriveData, isBucketListInitiation } from '../utils'
|
||||
|
||||
describe('online-drive utils', () => {
|
||||
describe('isBucketListInitiation', () => {
|
||||
it('should return true when data has buckets and no prefix/bucket set', () => {
|
||||
const data: OnlineDriveData[] = [
|
||||
{ bucket: 'bucket-1', files: [], is_truncated: false, next_page_parameters: {} },
|
||||
{ bucket: 'bucket-2', files: [], is_truncated: false, next_page_parameters: {} },
|
||||
]
|
||||
|
||||
expect(isBucketListInitiation(data, [], '')).toBe(true)
|
||||
})
|
||||
|
||||
it('should return false when bucket is already set', () => {
|
||||
const data: OnlineDriveData[] = [
|
||||
{ bucket: 'bucket-1', files: [], is_truncated: false, next_page_parameters: {} },
|
||||
]
|
||||
|
||||
expect(isBucketListInitiation(data, [], 'bucket-1')).toBe(false)
|
||||
})
|
||||
|
||||
it('should return false when prefix is set', () => {
|
||||
const data: OnlineDriveData[] = [
|
||||
{ bucket: 'bucket-1', files: [], is_truncated: false, next_page_parameters: {} },
|
||||
]
|
||||
|
||||
expect(isBucketListInitiation(data, ['folder/'], '')).toBe(false)
|
||||
})
|
||||
|
||||
it('should return false when single bucket has files', () => {
|
||||
const data: OnlineDriveData[] = [
|
||||
{
|
||||
bucket: 'bucket-1',
|
||||
files: [{ id: 'f1', name: 'test.txt', size: 100, type: 'file' as const }],
|
||||
is_truncated: false,
|
||||
next_page_parameters: {},
|
||||
},
|
||||
]
|
||||
|
||||
expect(isBucketListInitiation(data, [], '')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('convertOnlineDriveData', () => {
|
||||
it('should return empty result for empty data', () => {
|
||||
const result = convertOnlineDriveData([], [], '')
|
||||
expect(result.fileList).toEqual([])
|
||||
expect(result.isTruncated).toBe(false)
|
||||
expect(result.hasBucket).toBe(false)
|
||||
})
|
||||
|
||||
it('should convert bucket list initiation to bucket items', () => {
|
||||
const data: OnlineDriveData[] = [
|
||||
{ bucket: 'bucket-1', files: [], is_truncated: false, next_page_parameters: {} },
|
||||
{ bucket: 'bucket-2', files: [], is_truncated: false, next_page_parameters: {} },
|
||||
]
|
||||
|
||||
const result = convertOnlineDriveData(data, [], '')
|
||||
expect(result.fileList).toHaveLength(2)
|
||||
expect(result.fileList[0]).toEqual({
|
||||
id: 'bucket-1',
|
||||
name: 'bucket-1',
|
||||
type: OnlineDriveFileType.bucket,
|
||||
})
|
||||
expect(result.hasBucket).toBe(true)
|
||||
})
|
||||
|
||||
it('should convert files when not bucket list', () => {
|
||||
const data: OnlineDriveData[] = [
|
||||
{
|
||||
bucket: 'bucket-1',
|
||||
files: [
|
||||
{ id: 'f1', name: 'test.txt', size: 100, type: 'file' as const },
|
||||
{ id: 'f2', name: 'folder', size: 0, type: 'folder' as const },
|
||||
],
|
||||
is_truncated: true,
|
||||
next_page_parameters: { token: 'next' },
|
||||
},
|
||||
]
|
||||
|
||||
const result = convertOnlineDriveData(data, [], 'bucket-1')
|
||||
expect(result.fileList).toHaveLength(2)
|
||||
expect(result.fileList[0]!.type).toBe(OnlineDriveFileType.file)
|
||||
expect(result.fileList[0]!.size).toBe(100)
|
||||
expect(result.fileList[1]!.type).toBe(OnlineDriveFileType.folder)
|
||||
expect(result.fileList[1]!.size).toBeUndefined()
|
||||
expect(result.isTruncated).toBe(true)
|
||||
expect(result.nextPageParameters).toEqual({ token: 'next' })
|
||||
expect(result.hasBucket).toBe(true)
|
||||
})
|
||||
})
|
||||
})
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
import { act, fireEvent, render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import * as React from 'react'
|
||||
import FileList from '../index'
|
||||
|
||||
// Mock store - context provider requires mocking
|
||||
const mockStoreState = {
|
||||
setNextPageParameters: vi.fn(),
|
||||
currentNextPageParametersRef: { current: {} },
|
||||
isTruncated: { current: false },
|
||||
hasBucket: false,
|
||||
setOnlineDriveFileList: vi.fn(),
|
||||
setSelectedFileIds: vi.fn(),
|
||||
setBreadcrumbs: vi.fn(),
|
||||
setPrefix: vi.fn(),
|
||||
setBucket: vi.fn(),
|
||||
}
|
||||
|
||||
const mockGetState = vi.fn(() => mockStoreState)
|
||||
const mockDataSourceStore = { getState: mockGetState }
|
||||
|
||||
vi.mock('../../../store', () => ({
|
||||
useDataSourceStore: () => mockDataSourceStore,
|
||||
useDataSourceStoreWithSelector: (selector: (s: typeof mockStoreState) => unknown) =>
|
||||
selector(mockStoreState),
|
||||
}))
|
||||
|
||||
type FileListProps = React.ComponentProps<typeof FileList>
|
||||
|
||||
const createDefaultProps = (overrides?: Partial<FileListProps>): FileListProps => ({
|
||||
fileList: [],
|
||||
selectedFileIds: [],
|
||||
breadcrumbs: [],
|
||||
keywords: '',
|
||||
bucket: '',
|
||||
isInPipeline: false,
|
||||
resetKeywords: vi.fn(),
|
||||
updateKeywords: vi.fn(),
|
||||
searchResultsLength: 0,
|
||||
handleSelectFile: vi.fn(),
|
||||
handleOpenFolder: vi.fn(),
|
||||
isLoading: false,
|
||||
supportBatchUpload: true,
|
||||
...overrides,
|
||||
})
|
||||
|
||||
const resetMockStoreState = () => {
|
||||
mockStoreState.setNextPageParameters = vi.fn()
|
||||
mockStoreState.currentNextPageParametersRef = { current: {} }
|
||||
mockStoreState.isTruncated = { current: false }
|
||||
mockStoreState.hasBucket = false
|
||||
mockStoreState.setOnlineDriveFileList = vi.fn()
|
||||
mockStoreState.setSelectedFileIds = vi.fn()
|
||||
mockStoreState.setBreadcrumbs = vi.fn()
|
||||
mockStoreState.setPrefix = vi.fn()
|
||||
mockStoreState.setBucket = vi.fn()
|
||||
}
|
||||
|
||||
describe('FileList', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetMockStoreState()
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
describe('State Management', () => {
|
||||
describe('debounced keywords update', () => {
|
||||
it('should delay the search until typing has stopped', () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
const mockUpdateKeywords = vi.fn()
|
||||
const props = createDefaultProps({ updateKeywords: mockUpdateKeywords })
|
||||
render(<FileList {...props} />)
|
||||
const input = screen.getByPlaceholderText(
|
||||
'datasetPipeline.onlineDrive.breadcrumbs.searchPlaceholder',
|
||||
)
|
||||
|
||||
fireEvent.change(input, { target: { value: 'debounced-value' } })
|
||||
|
||||
expect(mockUpdateKeywords).not.toHaveBeenCalled()
|
||||
act(() => vi.advanceTimersByTime(499))
|
||||
expect(mockUpdateKeywords).not.toHaveBeenCalled()
|
||||
act(() => vi.advanceTimersByTime(1))
|
||||
expect(mockUpdateKeywords).toHaveBeenCalledExactlyOnceWith('debounced-value')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// Event Handlers Tests
|
||||
describe('Event Handlers', () => {
|
||||
describe('handleInputChange', () => {
|
||||
it('should search only for the latest input after rapid changes', () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
const mockUpdateKeywords = vi.fn()
|
||||
const props = createDefaultProps({ updateKeywords: mockUpdateKeywords })
|
||||
render(<FileList {...props} />)
|
||||
const input = screen.getByPlaceholderText(
|
||||
'datasetPipeline.onlineDrive.breadcrumbs.searchPlaceholder',
|
||||
)
|
||||
|
||||
fireEvent.change(input, { target: { value: 'a' } })
|
||||
fireEvent.change(input, { target: { value: 'ab' } })
|
||||
fireEvent.change(input, { target: { value: 'abc' } })
|
||||
|
||||
act(() => vi.advanceTimersByTime(500))
|
||||
expect(mockUpdateKeywords).toHaveBeenCalledExactlyOnceWith('abc')
|
||||
expect(input).toHaveValue('abc')
|
||||
})
|
||||
})
|
||||
|
||||
describe('handleResetKeywords', () => {
|
||||
it('should cancel the pending search before clearing the query', async () => {
|
||||
vi.useFakeTimers({ shouldAdvanceTime: true })
|
||||
const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime })
|
||||
const mockResetKeywords = vi.fn()
|
||||
const props = createDefaultProps({ resetKeywords: mockResetKeywords })
|
||||
render(<FileList {...props} />)
|
||||
const input = screen.getByRole('searchbox', {
|
||||
name: 'datasetPipeline.onlineDrive.breadcrumbs.searchPlaceholder',
|
||||
})
|
||||
|
||||
await user.type(input, 'report')
|
||||
await user.click(screen.getByRole('button', { name: 'common.operation.clear' }))
|
||||
|
||||
act(() => vi.advanceTimersByTime(500))
|
||||
expect(props.updateKeywords).not.toHaveBeenCalled()
|
||||
expect(mockResetKeywords).toHaveBeenCalledOnce()
|
||||
expect(input).toHaveValue('')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// File Type Variations
|
||||
|
||||
// Search Results Display
|
||||
|
||||
// Callback Stability
|
||||
})
|
||||
|
||||
it('clears an unmatched drive search from the empty-result action', async () => {
|
||||
const user = userEvent.setup()
|
||||
const props = createDefaultProps({ keywords: 'missing' })
|
||||
render(<FileList {...props} />)
|
||||
await user.click(
|
||||
screen.getByRole('button', { name: 'datasetPipeline.onlineDrive.resetKeywords' }),
|
||||
)
|
||||
expect(props.resetKeywords).toHaveBeenCalledOnce()
|
||||
})
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
import type { ComponentProps } from 'react'
|
||||
import { render, screen, within } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { createDataSourceStore } from '../../../../../store'
|
||||
import { DataSourceContext } from '../../../../../store/provider'
|
||||
import Breadcrumbs from '../index'
|
||||
|
||||
let store = createDataSourceStore()
|
||||
|
||||
function renderPath(overrides: Partial<ComponentProps<typeof Breadcrumbs>> = {}) {
|
||||
const props = {
|
||||
breadcrumbs: ['Documents', 'Reports'],
|
||||
keywords: '',
|
||||
bucket: '',
|
||||
searchResultsLength: 0,
|
||||
isInPipeline: false,
|
||||
...overrides,
|
||||
}
|
||||
store.setState({
|
||||
breadcrumbs: props.breadcrumbs,
|
||||
prefix: props.breadcrumbs.map((_, index) => `prefix-${index}`),
|
||||
bucket: props.bucket,
|
||||
selectedFileIds: ['selected-file'],
|
||||
onlineDriveFileList: [{ id: 'selected-file', name: 'report.txt', type: 'file', size: 10 }],
|
||||
})
|
||||
return render(<Breadcrumbs {...props} />, {
|
||||
wrapper: ({ children }) => <DataSourceContext value={store}>{children}</DataSourceContext>,
|
||||
})
|
||||
}
|
||||
|
||||
const allFiles = 'datasetPipeline.onlineDrive.breadcrumbs.allFiles'
|
||||
const allBuckets = 'datasetPipeline.onlineDrive.breadcrumbs.allBuckets'
|
||||
|
||||
beforeEach(() => {
|
||||
store = createDataSourceStore()
|
||||
})
|
||||
|
||||
it('exposes path items without counting decorative separators and marks the current directory', async () => {
|
||||
const user = userEvent.setup()
|
||||
renderPath()
|
||||
const path = within(screen.getByRole('navigation', { name: allFiles }))
|
||||
expect(path.getAllByRole('listitem')).toHaveLength(3)
|
||||
const current = path.getByRole('button', { name: 'Reports' })
|
||||
expect(current).toHaveAttribute('aria-current', 'location')
|
||||
expect(current).toBeDisabled()
|
||||
await user.click(current)
|
||||
expect(store.getState().breadcrumbs).toEqual(['Documents', 'Reports'])
|
||||
|
||||
await user.click(path.getByRole('button', { name: 'Documents' }))
|
||||
expect(store.getState().breadcrumbs).toEqual(['Documents'])
|
||||
expect(store.getState().prefix).toEqual(['prefix-0'])
|
||||
expect(store.getState().onlineDriveFileList).toEqual([])
|
||||
expect(store.getState().selectedFileIds).toEqual([])
|
||||
})
|
||||
|
||||
it('returns to the drive root and disables root navigation once already there', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { rerender } = renderPath()
|
||||
await user.click(screen.getByRole('button', { name: allFiles }))
|
||||
expect(store.getState().breadcrumbs).toEqual([])
|
||||
expect(store.getState().prefix).toEqual([])
|
||||
expect(store.getState().onlineDriveFileList).toEqual([])
|
||||
expect(store.getState().selectedFileIds).toEqual([])
|
||||
|
||||
rerender(
|
||||
<Breadcrumbs
|
||||
breadcrumbs={[]}
|
||||
keywords=""
|
||||
bucket=""
|
||||
searchResultsLength={0}
|
||||
isInPipeline={false}
|
||||
/>,
|
||||
)
|
||||
expect(screen.getAllByRole('listitem')).toHaveLength(1)
|
||||
expect(screen.getByRole('button', { name: allFiles })).toBeDisabled()
|
||||
expect(screen.getByRole('button', { name: allFiles })).toHaveAttribute('aria-current', 'location')
|
||||
})
|
||||
|
||||
it('distinguishes returning to the current bucket from leaving it for the bucket list', async () => {
|
||||
const user = userEvent.setup()
|
||||
store.setState({ hasBucket: true })
|
||||
renderPath({ bucket: 'Storage' })
|
||||
await user.click(screen.getByRole('button', { name: 'Storage' }))
|
||||
expect(store.getState().breadcrumbs).toEqual([])
|
||||
expect(store.getState().prefix).toEqual([])
|
||||
expect(store.getState().bucket).toBe('Storage')
|
||||
|
||||
await user.click(screen.getByRole('button', { name: allBuckets }))
|
||||
expect(store.getState().bucket).toEqual('')
|
||||
expect(store.getState().onlineDriveFileList).toEqual([])
|
||||
expect(store.getState().selectedFileIds).toEqual([])
|
||||
})
|
||||
|
||||
it('keeps the bucket-list action available at the current bucket root', () => {
|
||||
store.setState({ hasBucket: true })
|
||||
renderPath({ bucket: 'Storage', breadcrumbs: [] })
|
||||
expect(screen.getByRole('button', { name: allBuckets })).toBeEnabled()
|
||||
expect(screen.getByRole('button', { name: 'Storage' })).toBeDisabled()
|
||||
expect(screen.getByRole('button', { name: 'Storage' })).toHaveAttribute(
|
||||
'aria-current',
|
||||
'location',
|
||||
)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ isInPipeline: false, bucket: '', visiblePrefixes: ['A', 'B'] },
|
||||
{ isInPipeline: true, bucket: '', visiblePrefixes: ['A'] },
|
||||
{ isInPipeline: false, bucket: 'Storage', visiblePrefixes: ['A'] },
|
||||
{ isInPipeline: true, bucket: 'Storage', visiblePrefixes: [] },
|
||||
])(
|
||||
'keeps collapsed paths navigable with pipeline=$isInPipeline and bucket=$bucket',
|
||||
async ({ isInPipeline, bucket, visiblePrefixes }) => {
|
||||
const user = userEvent.setup()
|
||||
store.setState({ hasBucket: !!bucket })
|
||||
renderPath({ breadcrumbs: ['A', 'B', 'C', 'D'], isInPipeline, bucket })
|
||||
for (const name of visiblePrefixes)
|
||||
expect(screen.getByRole('button', { name })).toBeInTheDocument()
|
||||
expect(screen.queryByRole('button', { name: 'C' })).not.toBeInTheDocument()
|
||||
expect(screen.getByRole('button', { name: 'D' })).toHaveAttribute('aria-current', 'location')
|
||||
|
||||
const more = screen.getByRole('button', { name: 'common.operation.more' })
|
||||
more.focus()
|
||||
await user.keyboard('{Enter}')
|
||||
await user.click(await screen.findByRole('menuitem', { name: 'C' }))
|
||||
expect(store.getState().breadcrumbs).toEqual(['A', 'B', 'C'])
|
||||
expect(store.getState().prefix).toEqual(['prefix-0', 'prefix-1', 'prefix-2'])
|
||||
},
|
||||
)
|
||||
|
||||
it('shows a bucket-list heading instead of an empty navigation landmark', () => {
|
||||
store.setState({ hasBucket: true })
|
||||
renderPath({ breadcrumbs: [] })
|
||||
expect(screen.getByText(allBuckets)).toBeInTheDocument()
|
||||
expect(screen.queryByRole('navigation')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ breadcrumbs: [], bucket: 'Storage', folderName: 'Storage' },
|
||||
{ breadcrumbs: ['Documents', 'Reports'], bucket: 'Storage', folderName: 'Reports' },
|
||||
])(
|
||||
'replaces the path with search results scoped to $folderName',
|
||||
({ breadcrumbs, bucket, folderName }) => {
|
||||
renderPath({ breadcrumbs, bucket, keywords: 'query', searchResultsLength: 5 })
|
||||
expect(screen.getByText(/onlineDrive.breadcrumbs.searchResult/)).toHaveTextContent(folderName)
|
||||
expect(screen.queryByRole('navigation')).not.toBeInTheDocument()
|
||||
},
|
||||
)
|
||||
|
||||
it('keeps the directory path available when a search has no results', () => {
|
||||
renderPath({ keywords: 'query', searchResultsLength: 0 })
|
||||
expect(screen.getByRole('navigation', { name: allFiles })).toBeInTheDocument()
|
||||
expect(screen.getByRole('button', { name: 'Reports' })).toHaveAttribute(
|
||||
'aria-current',
|
||||
'location',
|
||||
)
|
||||
})
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
import { BreadcrumbItem, BreadcrumbSeparator } from '@langgenius/dify-ui/breadcrumb'
|
||||
import { IconButton } from '@langgenius/dify-ui/icon-button'
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from '@langgenius/dify-ui/tooltip'
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import DirectoryItem from './item'
|
||||
|
||||
type BucketProps = {
|
||||
bucketName: string
|
||||
current: boolean
|
||||
handleBackToBucketList: () => void
|
||||
handleClickBucketName: () => void
|
||||
}
|
||||
|
||||
const Bucket = ({
|
||||
bucketName,
|
||||
handleBackToBucketList,
|
||||
handleClickBucketName,
|
||||
current,
|
||||
}: BucketProps) => {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
const allBucketsLabel = t(($) => $['onlineDrive.breadcrumbs.allBuckets'], {
|
||||
ns: 'datasetPipeline',
|
||||
})
|
||||
|
||||
return (
|
||||
<>
|
||||
<BreadcrumbItem className="shrink-0">
|
||||
<Tooltip>
|
||||
<TooltipTrigger
|
||||
render={
|
||||
<IconButton
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="md"
|
||||
aria-label={allBucketsLabel}
|
||||
onClick={handleBackToBucketList}
|
||||
>
|
||||
<span
|
||||
aria-hidden
|
||||
className="i-custom-public-knowledge-online-drive-buckets-gray h-4.75 w-4.5"
|
||||
/>
|
||||
</IconButton>
|
||||
}
|
||||
/>
|
||||
<TooltipContent>{allBucketsLabel}</TooltipContent>
|
||||
</Tooltip>
|
||||
</BreadcrumbItem>
|
||||
<BreadcrumbSeparator className="system-xs-regular text-divider-deep" />
|
||||
<DirectoryItem
|
||||
name={bucketName}
|
||||
title={bucketName}
|
||||
current={current}
|
||||
onClick={handleClickBucketName}
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Bucket)
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import BreadcrumbItem from './item'
|
||||
|
||||
type DriveProps = {
|
||||
breadcrumbs: string[]
|
||||
handleBackToRoot: () => void
|
||||
}
|
||||
|
||||
export default function Drive({ breadcrumbs, handleBackToRoot }: DriveProps) {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
|
||||
return (
|
||||
<BreadcrumbItem
|
||||
name={t(($) => $['onlineDrive.breadcrumbs.allFiles'], { ns: 'datasetPipeline' })}
|
||||
current={breadcrumbs.length === 0}
|
||||
onClick={handleBackToRoot}
|
||||
/>
|
||||
)
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
import { Breadcrumb, BreadcrumbList } from '@langgenius/dify-ui/breadcrumb'
|
||||
import * as React from 'react'
|
||||
import { useCallback, useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../../../../store'
|
||||
import Bucket from './bucket'
|
||||
import Drive from './drive'
|
||||
import BreadcrumbItem from './item'
|
||||
import ActionsMenu from './menu'
|
||||
|
||||
type BreadcrumbsProps = {
|
||||
breadcrumbs: string[]
|
||||
keywords: string
|
||||
bucket: string
|
||||
searchResultsLength: number
|
||||
isInPipeline: boolean
|
||||
}
|
||||
|
||||
const Breadcrumbs = ({
|
||||
breadcrumbs,
|
||||
keywords,
|
||||
bucket,
|
||||
searchResultsLength,
|
||||
isInPipeline,
|
||||
}: BreadcrumbsProps) => {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
const dataSourceStore = useDataSourceStore()
|
||||
const hasBucket = useDataSourceStoreWithSelector((s) => s.hasBucket)
|
||||
const showSearchResult = !!keywords && searchResultsLength > 0
|
||||
const showBucketListTitle = breadcrumbs.length === 0 && hasBucket && bucket === ''
|
||||
|
||||
const displayBreadcrumbNum = useMemo(() => {
|
||||
const num = isInPipeline ? 2 : 3
|
||||
return bucket ? num - 1 : num
|
||||
}, [isInPipeline, bucket])
|
||||
|
||||
const breadcrumbsConfig = useMemo(() => {
|
||||
const prefixToDisplay = breadcrumbs.slice(0, displayBreadcrumbNum - 1)
|
||||
const collapsedBreadcrumbs = breadcrumbs.slice(displayBreadcrumbNum - 1, breadcrumbs.length - 1)
|
||||
return {
|
||||
original: breadcrumbs,
|
||||
needCollapsed: breadcrumbs.length > displayBreadcrumbNum,
|
||||
prefixBreadcrumbs: prefixToDisplay,
|
||||
collapsedBreadcrumbs,
|
||||
lastBreadcrumb: breadcrumbs[breadcrumbs.length - 1],
|
||||
}
|
||||
}, [displayBreadcrumbNum, breadcrumbs])
|
||||
|
||||
const handleBackToBucketList = useCallback(() => {
|
||||
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix, setBucket } =
|
||||
dataSourceStore.getState()
|
||||
setOnlineDriveFileList([])
|
||||
setSelectedFileIds([])
|
||||
setBucket('')
|
||||
setBreadcrumbs([])
|
||||
setPrefix([])
|
||||
}, [dataSourceStore])
|
||||
|
||||
const handleClickBucketName = useCallback(() => {
|
||||
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } =
|
||||
dataSourceStore.getState()
|
||||
setOnlineDriveFileList([])
|
||||
setSelectedFileIds([])
|
||||
setBreadcrumbs([])
|
||||
setPrefix([])
|
||||
}, [dataSourceStore])
|
||||
|
||||
const handleBackToRoot = useCallback(() => {
|
||||
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } =
|
||||
dataSourceStore.getState()
|
||||
setOnlineDriveFileList([])
|
||||
setSelectedFileIds([])
|
||||
setBreadcrumbs([])
|
||||
setPrefix([])
|
||||
}, [dataSourceStore])
|
||||
|
||||
const handleClickBreadcrumb = useCallback(
|
||||
(index: number) => {
|
||||
const {
|
||||
breadcrumbs,
|
||||
prefix,
|
||||
setOnlineDriveFileList,
|
||||
setSelectedFileIds,
|
||||
setBreadcrumbs,
|
||||
setPrefix,
|
||||
} = dataSourceStore.getState()
|
||||
const newBreadcrumbs = breadcrumbs.slice(0, index + 1)
|
||||
const newPrefix = prefix.slice(0, index + 1)
|
||||
setOnlineDriveFileList([])
|
||||
setSelectedFileIds([])
|
||||
setBreadcrumbs(newBreadcrumbs)
|
||||
setPrefix(newPrefix)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
return (
|
||||
<div className="flex min-w-0 grow items-center">
|
||||
{showSearchResult && (
|
||||
<div className="px-1.25 system-sm-medium">
|
||||
{t(($) => $['onlineDrive.breadcrumbs.searchResult'], {
|
||||
ns: 'datasetPipeline',
|
||||
searchResultsLength,
|
||||
folderName: breadcrumbs.at(-1) ?? bucket ?? '',
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
{!showSearchResult && showBucketListTitle && (
|
||||
<div className="px-1.25 system-sm-medium">
|
||||
{t(($) => $['onlineDrive.breadcrumbs.allBuckets'], { ns: 'datasetPipeline' })}
|
||||
</div>
|
||||
)}
|
||||
{!showSearchResult && !showBucketListTitle && (
|
||||
<Breadcrumb
|
||||
aria-label={t(($) => $['onlineDrive.breadcrumbs.allFiles'], { ns: 'datasetPipeline' })}
|
||||
className="w-full"
|
||||
>
|
||||
<BreadcrumbList className="gap-0.5">
|
||||
{hasBucket && bucket && (
|
||||
<Bucket
|
||||
bucketName={bucket}
|
||||
handleBackToBucketList={handleBackToBucketList}
|
||||
handleClickBucketName={handleClickBucketName}
|
||||
current={breadcrumbs.length === 0}
|
||||
/>
|
||||
)}
|
||||
{!hasBucket && <Drive breadcrumbs={breadcrumbs} handleBackToRoot={handleBackToRoot} />}
|
||||
{!breadcrumbsConfig.needCollapsed && (
|
||||
<>
|
||||
{breadcrumbsConfig.original.map((breadcrumb, index) => {
|
||||
const isLast = index === breadcrumbsConfig.original.length - 1
|
||||
return (
|
||||
<BreadcrumbItem
|
||||
key={breadcrumbs.slice(0, index + 1).join('/')}
|
||||
onClick={() => handleClickBreadcrumb(index)}
|
||||
name={breadcrumb}
|
||||
title={breadcrumb}
|
||||
current={isLast}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
</>
|
||||
)}
|
||||
{breadcrumbsConfig.needCollapsed && (
|
||||
<>
|
||||
{breadcrumbsConfig.prefixBreadcrumbs.map((breadcrumb, index) => {
|
||||
return (
|
||||
<BreadcrumbItem
|
||||
key={breadcrumbs.slice(0, index + 1).join('/')}
|
||||
onClick={() => handleClickBreadcrumb(index)}
|
||||
name={breadcrumb}
|
||||
title={breadcrumb}
|
||||
/>
|
||||
)
|
||||
})}
|
||||
<ActionsMenu
|
||||
startIndex={breadcrumbsConfig.prefixBreadcrumbs.length}
|
||||
breadcrumbs={breadcrumbsConfig.collapsedBreadcrumbs}
|
||||
onBreadcrumbClick={handleClickBreadcrumb}
|
||||
/>
|
||||
<BreadcrumbItem
|
||||
onClick={() => handleClickBreadcrumb(breadcrumbs.length - 1)}
|
||||
name={breadcrumbsConfig.lastBreadcrumb!}
|
||||
title={breadcrumbsConfig.lastBreadcrumb!}
|
||||
current
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</BreadcrumbList>
|
||||
</Breadcrumb>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Breadcrumbs)
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import { BreadcrumbSeparator, BreadcrumbItem as PathItem } from '@langgenius/dify-ui/breadcrumb'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
|
||||
type BreadcrumbItemProps = {
|
||||
name: string
|
||||
onClick: () => void
|
||||
current?: boolean
|
||||
title?: string
|
||||
}
|
||||
|
||||
export default function BreadcrumbItem({
|
||||
name,
|
||||
onClick,
|
||||
current = false,
|
||||
title,
|
||||
}: BreadcrumbItemProps) {
|
||||
return (
|
||||
<>
|
||||
<PathItem>
|
||||
<button
|
||||
type="button"
|
||||
aria-current={current ? 'location' : undefined}
|
||||
className={cn(
|
||||
'truncate rounded-md px-1.25 py-1 focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:outline-hidden',
|
||||
current
|
||||
? 'system-sm-medium text-text-secondary'
|
||||
: 'system-sm-regular text-text-tertiary hover:bg-state-base-hover',
|
||||
)}
|
||||
disabled={current}
|
||||
onClick={onClick}
|
||||
title={title}
|
||||
>
|
||||
{name}
|
||||
</button>
|
||||
</PathItem>
|
||||
{!current && <BreadcrumbSeparator className="system-xs-regular text-divider-deep" />}
|
||||
</>
|
||||
)
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
import { BreadcrumbItem, BreadcrumbSeparator } from '@langgenius/dify-ui/breadcrumb'
|
||||
import { IconButton } from '@langgenius/dify-ui/icon-button'
|
||||
import { Menu, MenuContent, MenuItem, MenuTrigger } from '@langgenius/dify-ui/menu'
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type ActionsMenuProps = {
|
||||
startIndex: number
|
||||
breadcrumbs: string[]
|
||||
onBreadcrumbClick: (index: number) => void
|
||||
}
|
||||
|
||||
const ActionsMenu = ({ startIndex, breadcrumbs, onBreadcrumbClick }: ActionsMenuProps) => {
|
||||
const { t } = useTranslation(['common'])
|
||||
|
||||
return (
|
||||
<>
|
||||
<BreadcrumbItem className="shrink-0">
|
||||
<Menu>
|
||||
<MenuTrigger
|
||||
render={
|
||||
<IconButton
|
||||
aria-label={t(($) => $['operation.more'], { ns: 'common' })}
|
||||
className="data-popup-open:bg-state-base-hover"
|
||||
>
|
||||
<span aria-hidden className="i-ri-more-fill size-4" />
|
||||
</IconButton>
|
||||
}
|
||||
/>
|
||||
<MenuContent align="start" className="w-34">
|
||||
{breadcrumbs.map((breadcrumb, index) => (
|
||||
<MenuItem
|
||||
key={breadcrumbs.slice(0, index + 1).join('/')}
|
||||
className="px-3 py-1.5 system-md-regular"
|
||||
onClick={() => onBreadcrumbClick(startIndex + index)}
|
||||
>
|
||||
{breadcrumb}
|
||||
</MenuItem>
|
||||
))}
|
||||
</MenuContent>
|
||||
</Menu>
|
||||
</BreadcrumbItem>
|
||||
<BreadcrumbSeparator className="system-xs-regular text-divider-deep" />
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(ActionsMenu)
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { SearchInput } from '@/app/components/base/search-input'
|
||||
import Breadcrumbs from './breadcrumbs'
|
||||
|
||||
type HeaderProps = {
|
||||
breadcrumbs: string[]
|
||||
inputValue: string
|
||||
keywords: string
|
||||
bucket: string
|
||||
searchResultsLength: number
|
||||
onSearchValueChange: (value: string) => void
|
||||
isInPipeline: boolean
|
||||
}
|
||||
|
||||
const Header = ({
|
||||
breadcrumbs,
|
||||
inputValue,
|
||||
keywords,
|
||||
bucket,
|
||||
isInPipeline,
|
||||
searchResultsLength,
|
||||
onSearchValueChange,
|
||||
}: HeaderProps) => {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
const searchLabel = t(($) => $['onlineDrive.breadcrumbs.searchPlaceholder'], {
|
||||
ns: 'datasetPipeline',
|
||||
})
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-x-2 bg-components-panel-bg p-1 pl-3">
|
||||
<Breadcrumbs
|
||||
breadcrumbs={breadcrumbs}
|
||||
keywords={keywords}
|
||||
bucket={bucket}
|
||||
searchResultsLength={searchResultsLength}
|
||||
isInPipeline={isInPipeline}
|
||||
/>
|
||||
<SearchInput
|
||||
className="h-8 w-50 shrink-0"
|
||||
value={inputValue}
|
||||
onValueChange={onSearchValueChange}
|
||||
aria-label={searchLabel}
|
||||
placeholder={searchLabel}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Header)
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import { useDebounceFn } from 'ahooks'
|
||||
import { useState } from 'react'
|
||||
import Header from './header'
|
||||
import List from './list'
|
||||
|
||||
type FileListProps = {
|
||||
fileList: OnlineDriveFile[]
|
||||
selectedFileIds: string[]
|
||||
breadcrumbs: string[]
|
||||
keywords: string
|
||||
bucket: string
|
||||
isInPipeline: boolean
|
||||
resetKeywords: () => void
|
||||
updateKeywords: (keywords: string) => void
|
||||
searchResultsLength: number
|
||||
handleSelectFile: (file: OnlineDriveFile) => void
|
||||
handleOpenFolder: (file: OnlineDriveFile) => void
|
||||
isLoading: boolean
|
||||
supportBatchUpload: boolean
|
||||
}
|
||||
|
||||
const FileList = ({
|
||||
fileList,
|
||||
selectedFileIds,
|
||||
breadcrumbs,
|
||||
keywords,
|
||||
bucket,
|
||||
resetKeywords,
|
||||
updateKeywords,
|
||||
searchResultsLength,
|
||||
handleSelectFile,
|
||||
handleOpenFolder,
|
||||
isInPipeline,
|
||||
isLoading,
|
||||
supportBatchUpload,
|
||||
}: FileListProps) => {
|
||||
const [inputValue, setInputValue] = useState(keywords)
|
||||
|
||||
const { run: updateKeywordsWithDebounce, cancel: cancelKeywordsUpdate } = useDebounceFn(
|
||||
(keywords: string) => {
|
||||
updateKeywords(keywords)
|
||||
},
|
||||
{ wait: 500 },
|
||||
)
|
||||
|
||||
const handleResetKeywords = () => {
|
||||
cancelKeywordsUpdate()
|
||||
setInputValue('')
|
||||
resetKeywords()
|
||||
}
|
||||
|
||||
const handleSearchValueChange = (keywords: string) => {
|
||||
if (!keywords) {
|
||||
handleResetKeywords()
|
||||
return
|
||||
}
|
||||
|
||||
setInputValue(keywords)
|
||||
updateKeywordsWithDebounce(keywords)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-100 flex-col overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg shadow-xs shadow-shadow-shadow-3">
|
||||
<Header
|
||||
breadcrumbs={breadcrumbs}
|
||||
inputValue={inputValue}
|
||||
keywords={keywords}
|
||||
bucket={bucket}
|
||||
isInPipeline={isInPipeline}
|
||||
onSearchValueChange={handleSearchValueChange}
|
||||
searchResultsLength={searchResultsLength}
|
||||
/>
|
||||
<List
|
||||
fileList={fileList}
|
||||
selectedFileIds={selectedFileIds}
|
||||
keywords={keywords}
|
||||
handleResetKeywords={handleResetKeywords}
|
||||
handleOpenFolder={handleOpenFolder}
|
||||
handleSelectFile={handleSelectFile}
|
||||
isLoading={isLoading}
|
||||
supportBatchUpload={supportBatchUpload}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default FileList
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
import type { ComponentProps } from 'react'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import { act, render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { OnlineDriveFileType } from '@/models/pipeline'
|
||||
import { createDataSourceStore } from '../../../../store'
|
||||
import { DataSourceContext } from '../../../../store/provider'
|
||||
import List from '../index'
|
||||
|
||||
const files: OnlineDriveFile[] = [
|
||||
{ id: 'first', name: 'First.pdf', type: OnlineDriveFileType.file, size: 1024 },
|
||||
{ id: 'second', name: 'Second.txt', type: OnlineDriveFileType.file, size: 512 },
|
||||
{ id: 'folder', name: 'Archive', type: OnlineDriveFileType.folder },
|
||||
]
|
||||
const observers: Observer[] = []
|
||||
class Observer implements IntersectionObserver {
|
||||
readonly root = null
|
||||
readonly rootMargin = '100px'
|
||||
readonly scrollMargin = '0px'
|
||||
readonly thresholds = [0]
|
||||
observe = vi.fn()
|
||||
unobserve = vi.fn()
|
||||
disconnect = vi.fn()
|
||||
takeRecords = () => []
|
||||
private callback: IntersectionObserverCallback
|
||||
constructor(callback: IntersectionObserverCallback) {
|
||||
this.callback = callback
|
||||
observers.push(this)
|
||||
}
|
||||
intersect(visible: boolean) {
|
||||
const target = this.observe.mock.lastCall![0] as Element
|
||||
const rect = target.getBoundingClientRect()
|
||||
this.callback(
|
||||
[
|
||||
{
|
||||
isIntersecting: visible,
|
||||
intersectionRatio: visible ? 1 : 0,
|
||||
target,
|
||||
time: 0,
|
||||
rootBounds: null,
|
||||
boundingClientRect: rect,
|
||||
intersectionRect: rect,
|
||||
},
|
||||
],
|
||||
this,
|
||||
)
|
||||
}
|
||||
}
|
||||
function setup(overrides: Partial<ComponentProps<typeof List>> = {}) {
|
||||
const store = createDataSourceStore()
|
||||
const props = {
|
||||
fileList: files,
|
||||
selectedFileIds: ['first'],
|
||||
keywords: '',
|
||||
isLoading: false,
|
||||
supportBatchUpload: true,
|
||||
handleResetKeywords: vi.fn(),
|
||||
handleSelectFile: vi.fn(),
|
||||
handleOpenFolder: vi.fn(),
|
||||
...overrides,
|
||||
}
|
||||
const view = render(
|
||||
<DataSourceContext value={store}>
|
||||
<List {...props} />
|
||||
</DataSourceContext>,
|
||||
)
|
||||
return {
|
||||
...view,
|
||||
store,
|
||||
props,
|
||||
update: (next: Partial<typeof props>) =>
|
||||
view.rerender(
|
||||
<DataSourceContext value={store}>
|
||||
<List {...props} {...next} />
|
||||
</DataSourceContext>,
|
||||
),
|
||||
}
|
||||
}
|
||||
beforeEach(() => {
|
||||
observers.length = 0
|
||||
vi.stubGlobal('IntersectionObserver', Observer)
|
||||
})
|
||||
afterEach(() => vi.unstubAllGlobals())
|
||||
|
||||
it.each([true, false])('reflects and updates selected files in batch=%s mode', async (multiple) => {
|
||||
const user = userEvent.setup()
|
||||
const { props, update } = setup({ supportBatchUpload: multiple })
|
||||
const role = multiple ? 'checkbox' : 'radio'
|
||||
expect(screen.getByRole(role, { name: 'First.pdf' })).toBeChecked()
|
||||
expect(screen.getByRole(role, { name: 'Second.txt' })).not.toBeChecked()
|
||||
await user.click(screen.getByRole(role, { name: 'Second.txt' }))
|
||||
expect(props.handleSelectFile).toHaveBeenCalledExactlyOnceWith(files[1])
|
||||
expect(props.handleOpenFolder).not.toHaveBeenCalled()
|
||||
update({ selectedFileIds: ['second'] })
|
||||
expect(screen.getByRole(role, { name: 'First.pdf' })).not.toBeChecked()
|
||||
expect(screen.getByRole(role, { name: 'Second.txt' })).toBeChecked()
|
||||
})
|
||||
|
||||
it('opens a folder without selecting it', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { props } = setup()
|
||||
await user.click(screen.getByText('Archive'))
|
||||
expect(props.handleOpenFolder).toHaveBeenCalledExactlyOnceWith(files[2])
|
||||
expect(props.handleSelectFile).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('distinguishes an empty folder from a failed search and allows resetting the search', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { update, props } = setup({ fileList: [], isLoading: true })
|
||||
expect(screen.queryByText('datasetPipeline.onlineDrive.emptyFolder')).not.toBeInTheDocument()
|
||||
update({ fileList: [], isLoading: false })
|
||||
expect(screen.getByText('datasetPipeline.onlineDrive.emptyFolder')).toBeInTheDocument()
|
||||
update({ fileList: [], keywords: 'missing', isLoading: false })
|
||||
expect(screen.getByText('datasetPipeline.onlineDrive.emptySearchResult')).toBeInTheDocument()
|
||||
expect(screen.queryByText('datasetPipeline.onlineDrive.emptyFolder')).not.toBeInTheDocument()
|
||||
await user.click(
|
||||
screen.getByRole('button', { name: 'datasetPipeline.onlineDrive.resetKeywords' }),
|
||||
)
|
||||
expect(props.handleResetKeywords).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('keeps loaded files visible while loading more and displays the appended page', () => {
|
||||
const { update } = setup({ isLoading: true })
|
||||
expect(screen.getByRole('checkbox', { name: 'First.pdf' })).toBeInTheDocument()
|
||||
expect(screen.getByRole('status', { name: 'appApi.loading' })).toBeInTheDocument()
|
||||
update({
|
||||
isLoading: false,
|
||||
fileList: [...files, { id: 'third', name: 'Third.pdf', type: OnlineDriveFileType.file }],
|
||||
})
|
||||
expect(screen.queryByRole('status', { name: 'appApi.loading' })).not.toBeInTheDocument()
|
||||
expect(screen.getByRole('checkbox', { name: 'Third.pdf' })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it.each([
|
||||
{
|
||||
visible: true,
|
||||
truncated: true,
|
||||
loading: false,
|
||||
expected: { cursor: 'next', bucket: 'archive' },
|
||||
},
|
||||
{ visible: false, truncated: true, loading: false, expected: {} },
|
||||
{ visible: true, truncated: false, loading: false, expected: {} },
|
||||
{ visible: true, truncated: true, loading: true, expected: {} },
|
||||
])(
|
||||
'loads another page only when visible, truncated, and idle: %j',
|
||||
({ visible, truncated, loading, expected }) => {
|
||||
const { store } = setup({ isLoading: loading })
|
||||
store.getState().isTruncated.current = truncated
|
||||
store.getState().currentNextPageParametersRef.current = { cursor: 'next', bucket: 'archive' }
|
||||
act(() => observers[0]!.intersect(visible))
|
||||
expect(store.getState().nextPageParameters).toEqual(expected)
|
||||
},
|
||||
)
|
||||
|
||||
it('replaces the observer when loading changes and disconnects the current observer on unmount', () => {
|
||||
const { update, unmount, store } = setup({ isLoading: true })
|
||||
const first = observers[0]!
|
||||
update({ isLoading: false })
|
||||
expect(first.disconnect).toHaveBeenCalledOnce()
|
||||
const current = observers[1]!
|
||||
store.getState().isTruncated.current = true
|
||||
store.getState().currentNextPageParametersRef.current = { cursor: 'latest' }
|
||||
act(() => current.intersect(true))
|
||||
expect(store.getState().nextPageParameters).toEqual({ cursor: 'latest' })
|
||||
unmount()
|
||||
expect(current.disconnect).toHaveBeenCalledOnce()
|
||||
})
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import { RadioGroup } from '@langgenius/dify-ui/radio-group'
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import Item from '../item'
|
||||
|
||||
describe('Item', () => {
|
||||
const makeFile = (
|
||||
type: OnlineDriveFile['type'],
|
||||
name = 'test.pdf',
|
||||
size = 1024,
|
||||
): OnlineDriveFile => ({
|
||||
id: 'f-1',
|
||||
name,
|
||||
type,
|
||||
size,
|
||||
})
|
||||
|
||||
const defaultProps = {
|
||||
file: makeFile('file'),
|
||||
isSelected: false,
|
||||
onSelect: vi.fn(),
|
||||
onOpen: vi.fn(),
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('should render radio for file type in single choice mode', () => {
|
||||
render(
|
||||
<RadioGroup aria-label="Files" value={defaultProps.file.id}>
|
||||
<Item {...defaultProps} isMultipleChoice={false} />
|
||||
</RadioGroup>,
|
||||
)
|
||||
|
||||
expect(screen.getByRole('radio', { name: 'test.pdf' })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('should not render checkbox for bucket type', () => {
|
||||
render(<Item {...defaultProps} file={makeFile('bucket', 'my-bucket')} />)
|
||||
expect(screen.queryByRole('checkbox')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('should call onOpen for folder click', async () => {
|
||||
const user = userEvent.setup()
|
||||
const file = makeFile('folder', 'my-folder')
|
||||
render(<Item {...defaultProps} file={file} />)
|
||||
await user.click(screen.getByText('my-folder'))
|
||||
expect(defaultProps.onOpen).toHaveBeenCalledWith(file)
|
||||
})
|
||||
|
||||
it('should call onSelect for file click', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(<Item {...defaultProps} />)
|
||||
await user.click(screen.getByText('test.pdf'))
|
||||
expect(defaultProps.onSelect).toHaveBeenCalledWith(defaultProps.file)
|
||||
})
|
||||
|
||||
it('should call onSelect once without bubbling to row click when checkbox is clicked', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(<Item {...defaultProps} />)
|
||||
|
||||
await user.click(screen.getByRole('checkbox', { name: 'test.pdf' }))
|
||||
|
||||
expect(defaultProps.onSelect).toHaveBeenCalledTimes(1)
|
||||
expect(defaultProps.onSelect).toHaveBeenCalledWith(defaultProps.file)
|
||||
expect(defaultProps.onOpen).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('should not call handlers when disabled', async () => {
|
||||
const user = userEvent.setup()
|
||||
render(<Item {...defaultProps} disabled={true} />)
|
||||
await user.click(screen.getByText('test.pdf'))
|
||||
expect(defaultProps.onSelect).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
import { describe, expect, it } from 'vite-plus/test'
|
||||
import { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader/types'
|
||||
import { getFileExtension, getFileType } from '../utils'
|
||||
|
||||
describe('getFileExtension', () => {
|
||||
it('should return extension for normal file', () => {
|
||||
expect(getFileExtension('test.pdf')).toBe('pdf')
|
||||
})
|
||||
|
||||
it('should return lowercase extension', () => {
|
||||
expect(getFileExtension('test.PDF')).toBe('pdf')
|
||||
})
|
||||
|
||||
it('should return last extension for multiple dots', () => {
|
||||
expect(getFileExtension('my.file.name.txt')).toBe('txt')
|
||||
})
|
||||
|
||||
it('should return empty string for no extension', () => {
|
||||
expect(getFileExtension('noext')).toBe('')
|
||||
})
|
||||
|
||||
it('should return empty string for empty string', () => {
|
||||
expect(getFileExtension('')).toBe('')
|
||||
})
|
||||
|
||||
it('should return empty string for dotfile with no extension', () => {
|
||||
expect(getFileExtension('.gitignore')).toBe('')
|
||||
})
|
||||
})
|
||||
|
||||
describe('getFileType', () => {
|
||||
it('should return pdf for .pdf files', () => {
|
||||
expect(getFileType('doc.pdf')).toBe(FileAppearanceTypeEnum.pdf)
|
||||
})
|
||||
|
||||
it('should return markdown for .md files', () => {
|
||||
expect(getFileType('readme.md')).toBe(FileAppearanceTypeEnum.markdown)
|
||||
})
|
||||
|
||||
it('should return markdown for .mdx files', () => {
|
||||
expect(getFileType('page.mdx')).toBe(FileAppearanceTypeEnum.markdown)
|
||||
})
|
||||
|
||||
it('should return excel for .xlsx files', () => {
|
||||
expect(getFileType('data.xlsx')).toBe(FileAppearanceTypeEnum.excel)
|
||||
})
|
||||
|
||||
it('should return excel for .csv files', () => {
|
||||
expect(getFileType('data.csv')).toBe(FileAppearanceTypeEnum.excel)
|
||||
})
|
||||
|
||||
it('should return word for .docx files', () => {
|
||||
expect(getFileType('doc.docx')).toBe(FileAppearanceTypeEnum.word)
|
||||
})
|
||||
|
||||
it('should return ppt for .pptx files', () => {
|
||||
expect(getFileType('slides.pptx')).toBe(FileAppearanceTypeEnum.ppt)
|
||||
})
|
||||
|
||||
it('should return code for .html files', () => {
|
||||
expect(getFileType('page.html')).toBe(FileAppearanceTypeEnum.code)
|
||||
})
|
||||
|
||||
it('should return code for .json files', () => {
|
||||
expect(getFileType('config.json')).toBe(FileAppearanceTypeEnum.code)
|
||||
})
|
||||
|
||||
it('should return gif for .gif files', () => {
|
||||
expect(getFileType('animation.gif')).toBe(FileAppearanceTypeEnum.gif)
|
||||
})
|
||||
|
||||
it('should return custom for unknown extension', () => {
|
||||
expect(getFileType('file.xyz')).toBe(FileAppearanceTypeEnum.custom)
|
||||
})
|
||||
|
||||
it('should return custom for no extension', () => {
|
||||
expect(getFileType('noext')).toBe(FileAppearanceTypeEnum.custom)
|
||||
})
|
||||
})
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
const EmptyFolder = () => {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
|
||||
return (
|
||||
<div className="flex size-full items-center justify-center rounded-[10px] bg-background-section px-1 py-1.5">
|
||||
<span className="system-xs-regular text-text-tertiary">
|
||||
{t(($) => $['onlineDrive.emptyFolder'], { ns: 'datasetPipeline' })}
|
||||
</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(EmptyFolder)
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type EmptySearchResultProps = {
|
||||
onResetKeywords: () => void
|
||||
}
|
||||
|
||||
const EmptySearchResult = ({
|
||||
onResetKeywords,
|
||||
}: EmptySearchResultProps & {
|
||||
className?: string
|
||||
}) => {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
|
||||
return (
|
||||
<div className="flex size-full flex-col items-center justify-center gap-y-2 rounded-[10px] bg-background-section p-6">
|
||||
<span
|
||||
aria-hidden
|
||||
className="i-custom-vender-knowledge-search-menu size-8 text-text-tertiary"
|
||||
/>
|
||||
<div className="system-sm-regular text-text-secondary">
|
||||
{t(($) => $['onlineDrive.emptySearchResult'], { ns: 'datasetPipeline' })}
|
||||
</div>
|
||||
<Button variant="secondary-accent" size="small" onClick={onResetKeywords}>
|
||||
<span>{t(($) => $['onlineDrive.resetKeywords'], { ns: 'datasetPipeline' })}</span>
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(EmptySearchResult)
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import * as React from 'react'
|
||||
import { useMemo } from 'react'
|
||||
import FileTypeIcon from '@/app/components/base/file-uploader/file-type-icon'
|
||||
import { OnlineDriveFileType } from '@/models/pipeline'
|
||||
import { getFileType } from './utils'
|
||||
|
||||
type FileIconProps = {
|
||||
type: OnlineDriveFileType
|
||||
fileName: string
|
||||
size?: 'sm' | 'md' | 'lg' | 'xl'
|
||||
className?: string
|
||||
}
|
||||
|
||||
const FileIcon = ({ type, fileName, size = 'md', className }: FileIconProps) => {
|
||||
const fileType = useMemo(() => {
|
||||
if (type === OnlineDriveFileType.bucket || type === OnlineDriveFileType.folder) return 'custom'
|
||||
|
||||
return getFileType(fileName)
|
||||
}, [type, fileName])
|
||||
|
||||
if (type === OnlineDriveFileType.bucket) {
|
||||
return (
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn(
|
||||
'i-custom-public-knowledge-online-drive-buckets-blue h-5.25 w-5',
|
||||
cn('size-4.5', className),
|
||||
)}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
if (type === OnlineDriveFileType.folder) {
|
||||
return (
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn(
|
||||
'i-custom-public-knowledge-online-drive-folder h-4.75 w-5',
|
||||
cn('size-4.5', className),
|
||||
)}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
return <FileTypeIcon size={size} type={fileType} className={cn('size-4.5', className)} />
|
||||
}
|
||||
|
||||
export default React.memo(FileIcon)
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import { RadioGroup } from '@langgenius/dify-ui/radio-group'
|
||||
import { RiLoader2Line } from '@remixicon/react'
|
||||
import * as React from 'react'
|
||||
import { useEffect, useRef } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { LoadingPlaceholder } from '@/app/components/base/loading-placeholder'
|
||||
import { useDataSourceStore } from '../../../store'
|
||||
import EmptyFolder from './empty-folder'
|
||||
import EmptySearchResult from './empty-search-result'
|
||||
import Item from './item'
|
||||
|
||||
type FileListProps = {
|
||||
fileList: OnlineDriveFile[]
|
||||
selectedFileIds: string[]
|
||||
keywords: string
|
||||
isLoading: boolean
|
||||
supportBatchUpload: boolean
|
||||
handleResetKeywords: () => void
|
||||
handleSelectFile: (file: OnlineDriveFile) => void
|
||||
handleOpenFolder: (file: OnlineDriveFile) => void
|
||||
}
|
||||
|
||||
const List = ({
|
||||
fileList,
|
||||
selectedFileIds,
|
||||
keywords,
|
||||
handleResetKeywords,
|
||||
handleSelectFile,
|
||||
handleOpenFolder,
|
||||
isLoading,
|
||||
supportBatchUpload,
|
||||
}: FileListProps) => {
|
||||
const { t } = useTranslation(['appApi', 'datasetPipeline'])
|
||||
const anchorRef = useRef<HTMLDivElement>(null)
|
||||
const observerRef = useRef<IntersectionObserver>(null)
|
||||
const dataSourceStore = useDataSourceStore()
|
||||
|
||||
useEffect(() => {
|
||||
if (anchorRef.current) {
|
||||
observerRef.current = new IntersectionObserver(
|
||||
(entries) => {
|
||||
const { setNextPageParameters, currentNextPageParametersRef, isTruncated } =
|
||||
dataSourceStore.getState()
|
||||
if (entries[0]!.isIntersecting && isTruncated.current && !isLoading)
|
||||
setNextPageParameters(currentNextPageParametersRef.current)
|
||||
},
|
||||
{
|
||||
rootMargin: '100px',
|
||||
},
|
||||
)
|
||||
observerRef.current.observe(anchorRef.current)
|
||||
}
|
||||
return () => observerRef.current?.disconnect()
|
||||
}, [anchorRef, isLoading, dataSourceStore])
|
||||
|
||||
const isAllLoading = isLoading && fileList.length === 0 && keywords.length === 0
|
||||
const isPartialLoading = isLoading && fileList.length > 0
|
||||
const isEmptyFolder = !isLoading && fileList.length === 0 && keywords.length === 0
|
||||
const isSearchResultEmpty = !isLoading && fileList.length === 0 && keywords.length > 0
|
||||
const selectedFileId = selectedFileIds[0]
|
||||
const handleRadioChange = (fileId: string) => {
|
||||
const selectedFile = fileList.find((file) => file.id === fileId)
|
||||
if (selectedFile) handleSelectFile(selectedFile)
|
||||
}
|
||||
const fileItems = fileList.map((file) => {
|
||||
const isSelected = selectedFileIds.includes(file.id)
|
||||
return (
|
||||
<Item
|
||||
key={file.id}
|
||||
file={file}
|
||||
isSelected={isSelected}
|
||||
onSelect={handleSelectFile}
|
||||
onOpen={handleOpenFolder}
|
||||
isMultipleChoice={supportBatchUpload}
|
||||
/>
|
||||
)
|
||||
})
|
||||
|
||||
return (
|
||||
<div className="grow overflow-hidden p-1 pt-0">
|
||||
{isAllLoading && <LoadingPlaceholder className="h-full" />}
|
||||
{isEmptyFolder && <EmptyFolder />}
|
||||
{isSearchResultEmpty && <EmptySearchResult onResetKeywords={handleResetKeywords} />}
|
||||
{fileList.length > 0 && (
|
||||
<div className="flex h-full flex-col gap-y-px overflow-y-auto rounded-[10px] bg-background-section px-1 py-1.5">
|
||||
{supportBatchUpload ? (
|
||||
fileItems
|
||||
) : (
|
||||
<RadioGroup
|
||||
aria-label={t(($) => $['onlineDrive.breadcrumbs.allFiles'], {
|
||||
ns: 'datasetPipeline',
|
||||
})}
|
||||
value={selectedFileId}
|
||||
onValueChange={handleRadioChange}
|
||||
className="contents"
|
||||
>
|
||||
{fileItems}
|
||||
</RadioGroup>
|
||||
)}
|
||||
{isPartialLoading && (
|
||||
<div
|
||||
className="flex items-center justify-center py-2"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
aria-label={t(($) => $.loading, { ns: 'appApi' })}
|
||||
>
|
||||
<RiLoader2Line className="size-4 text-text-tertiary" />
|
||||
</div>
|
||||
)}
|
||||
<div ref={anchorRef} className="h-0" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(List)
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import { Checkbox } from '@langgenius/dify-ui/checkbox'
|
||||
import { Popover, PopoverContent, PopoverTrigger } from '@langgenius/dify-ui/popover'
|
||||
import { Radio } from '@langgenius/dify-ui/radio-group'
|
||||
import * as React from 'react'
|
||||
import { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { formatFileSize } from '@/utils/format'
|
||||
import FileIcon from './file-icon'
|
||||
|
||||
type ItemProps = {
|
||||
file: OnlineDriveFile
|
||||
isSelected: boolean
|
||||
disabled?: boolean
|
||||
isMultipleChoice?: boolean
|
||||
onSelect: (file: OnlineDriveFile) => void
|
||||
onOpen: (file: OnlineDriveFile) => void
|
||||
}
|
||||
|
||||
const Item = ({
|
||||
file,
|
||||
isSelected,
|
||||
disabled = false,
|
||||
isMultipleChoice = true,
|
||||
onSelect,
|
||||
onOpen,
|
||||
}: ItemProps) => {
|
||||
const { t } = useTranslation(['datasetPipeline'])
|
||||
const { name, type, size } = file
|
||||
|
||||
const isBucket = type === 'bucket'
|
||||
const isFolder = type === 'folder'
|
||||
|
||||
const disabledTip = t(($) => $['onlineDrive.notSupportedFileType'], { ns: 'datasetPipeline' })
|
||||
|
||||
const handleCheckboxSelect = useCallback(() => {
|
||||
onSelect(file)
|
||||
}, [file, onSelect])
|
||||
|
||||
const handleClickItem = useCallback(
|
||||
(e: React.MouseEvent<HTMLDivElement>) => {
|
||||
e.stopPropagation()
|
||||
if (disabled) return
|
||||
if (isBucket || isFolder) {
|
||||
onOpen(file)
|
||||
return
|
||||
}
|
||||
onSelect(file)
|
||||
},
|
||||
[disabled, file, isBucket, isFolder, onOpen, onSelect],
|
||||
)
|
||||
|
||||
return (
|
||||
<div
|
||||
className="flex cursor-pointer items-center gap-2 rounded-md px-2 py-0.75 hover:bg-state-base-hover"
|
||||
onClick={handleClickItem}
|
||||
>
|
||||
{!isBucket && isMultipleChoice && (
|
||||
<span onClick={(event) => event.stopPropagation()}>
|
||||
<Checkbox
|
||||
className="shrink-0"
|
||||
disabled={disabled}
|
||||
checked={isSelected}
|
||||
aria-label={name}
|
||||
onCheckedChange={() => handleCheckboxSelect()}
|
||||
/>
|
||||
</span>
|
||||
)}
|
||||
{!isBucket && !isMultipleChoice && (
|
||||
<span onClick={(event) => event.stopPropagation()}>
|
||||
<Radio className="shrink-0" disabled={disabled} value={file.id} aria-label={name} />
|
||||
</span>
|
||||
)}
|
||||
{disabled ? (
|
||||
<Popover>
|
||||
<PopoverTrigger
|
||||
openOnHover
|
||||
aria-label={disabledTip}
|
||||
className="flex grow items-center gap-x-1 overflow-hidden border-0 bg-transparent p-0 py-0.5 text-left opacity-30"
|
||||
>
|
||||
<FileIcon type={type} fileName={name} className="shrink-0 transform-gpu" />
|
||||
<span className="grow truncate system-sm-medium text-text-secondary" title={name}>
|
||||
{name}
|
||||
</span>
|
||||
{!isFolder && typeof size === 'number' && (
|
||||
<span className="shrink-0 system-xs-regular text-text-tertiary">
|
||||
{formatFileSize(size)}
|
||||
</span>
|
||||
)}
|
||||
</PopoverTrigger>
|
||||
<PopoverContent
|
||||
side="top"
|
||||
align="end"
|
||||
className="px-3 py-2 system-xs-regular text-text-tertiary"
|
||||
>
|
||||
{disabledTip}
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
) : (
|
||||
<div className="flex grow items-center gap-x-1 overflow-hidden py-0.5">
|
||||
<FileIcon type={type} fileName={name} className="shrink-0 transform-gpu" />
|
||||
<span className="grow truncate system-sm-medium text-text-secondary" title={name}>
|
||||
{name}
|
||||
</span>
|
||||
{!isFolder && typeof size === 'number' && (
|
||||
<span className="shrink-0 system-xs-regular text-text-tertiary">
|
||||
{formatFileSize(size)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Item)
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
import { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader/types'
|
||||
import { FILE_EXTS } from '@/app/components/base/prompt-editor/constants'
|
||||
|
||||
export const getFileExtension = (fileName: string): string => {
|
||||
if (!fileName) return ''
|
||||
const parts = fileName.split('.')
|
||||
if (parts.length <= 1 || (parts[0] === '' && parts.length === 2)) return ''
|
||||
|
||||
return parts[parts.length - 1]!.toLowerCase()
|
||||
}
|
||||
|
||||
export const getFileType = (fileName: string) => {
|
||||
const extension = getFileExtension(fileName)
|
||||
|
||||
if (extension === 'gif') return FileAppearanceTypeEnum.gif
|
||||
|
||||
if (FILE_EXTS.image!.includes(extension.toUpperCase())) return FileAppearanceTypeEnum.image
|
||||
|
||||
if (FILE_EXTS.video!.includes(extension.toUpperCase())) return FileAppearanceTypeEnum.video
|
||||
|
||||
if (FILE_EXTS.audio!.includes(extension.toUpperCase())) return FileAppearanceTypeEnum.audio
|
||||
|
||||
if (extension === 'html' || extension === 'htm' || extension === 'xml' || extension === 'json')
|
||||
return FileAppearanceTypeEnum.code
|
||||
|
||||
if (extension === 'pdf') return FileAppearanceTypeEnum.pdf
|
||||
|
||||
if (extension === 'md' || extension === 'markdown' || extension === 'mdx')
|
||||
return FileAppearanceTypeEnum.markdown
|
||||
|
||||
if (extension === 'xlsx' || extension === 'xls' || extension === 'csv')
|
||||
return FileAppearanceTypeEnum.excel
|
||||
|
||||
if (extension === 'docx' || extension === 'doc') return FileAppearanceTypeEnum.word
|
||||
|
||||
if (extension === 'pptx' || extension === 'ppt') return FileAppearanceTypeEnum.ppt
|
||||
|
||||
if (FILE_EXTS.document!.includes(extension.toUpperCase())) return FileAppearanceTypeEnum.document
|
||||
|
||||
return FileAppearanceTypeEnum.custom
|
||||
}
|
||||
+243
@@ -0,0 +1,243 @@
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { produce } from 'immer'
|
||||
import { useQueryState } from 'nuqs'
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useShallow } from 'zustand/react/shallow'
|
||||
import {
|
||||
settingsQueryParamName,
|
||||
settingsQueryParser,
|
||||
} from '@/app/components/header/account-setting/query-params'
|
||||
import { toast } from '@/app/notifications'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { useDocLink } from '@/context/i18n'
|
||||
import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import Header from '../base/header'
|
||||
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
|
||||
import FileList from './file-list'
|
||||
import { convertOnlineDriveData } from './utils'
|
||||
|
||||
type OnlineDriveProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
onCredentialChange: (credentialId: string) => void
|
||||
isInPipeline?: boolean
|
||||
supportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const OnlineDrive = ({
|
||||
nodeId,
|
||||
nodeData,
|
||||
isInPipeline = false,
|
||||
supportBatchUpload = true,
|
||||
onCredentialChange,
|
||||
}: OnlineDriveProps) => {
|
||||
const docLink = useDocLink()
|
||||
const [isInitialMount, setIsInitialMount] = useState(true)
|
||||
const pipelineId = useDatasetDetailContextWithSelector((s) => s.dataset?.pipeline_id)
|
||||
const [, setSettingsDestination] = useQueryState(settingsQueryParamName, settingsQueryParser)
|
||||
const {
|
||||
nextPageParameters,
|
||||
breadcrumbs,
|
||||
prefix,
|
||||
keywords,
|
||||
bucket,
|
||||
selectedFileIds,
|
||||
onlineDriveFileList,
|
||||
currentCredentialId,
|
||||
} = useDataSourceStoreWithSelector(
|
||||
useShallow((state) => ({
|
||||
nextPageParameters: state.nextPageParameters,
|
||||
breadcrumbs: state.breadcrumbs,
|
||||
prefix: state.prefix,
|
||||
keywords: state.keywords,
|
||||
bucket: state.bucket,
|
||||
selectedFileIds: state.selectedFileIds,
|
||||
onlineDriveFileList: state.onlineDriveFileList,
|
||||
currentCredentialId: state.currentCredentialId,
|
||||
})),
|
||||
)
|
||||
const dataSourceStore = useDataSourceStore()
|
||||
const [isLoading, setIsLoading] = useState(false)
|
||||
const isLoadingRef = useRef(false)
|
||||
|
||||
const { data: dataSourceAuth } = useQuery(
|
||||
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
|
||||
input: { params: { provider_id: `${nodeData.plugin_id}/${nodeData.provider_name}` } },
|
||||
}),
|
||||
)
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const getOnlineDriveFiles = useCallback(async () => {
|
||||
if (isLoadingRef.current) return
|
||||
const { nextPageParameters, prefix, bucket, onlineDriveFileList, currentCredentialId } =
|
||||
dataSourceStore.getState()
|
||||
setIsLoading(true)
|
||||
isLoadingRef.current = true
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: {
|
||||
prefix: prefix[prefix.length - 1],
|
||||
bucket,
|
||||
next_page_parameters: nextPageParameters,
|
||||
max_keys: 30,
|
||||
},
|
||||
datasource_type: DatasourceType.onlineDrive,
|
||||
credential_id: currentCredentialId,
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
|
||||
const {
|
||||
setOnlineDriveFileList,
|
||||
isTruncated,
|
||||
currentNextPageParametersRef,
|
||||
setHasBucket,
|
||||
} = dataSourceStore.getState()
|
||||
const {
|
||||
fileList: newFileList,
|
||||
isTruncated: newIsTruncated,
|
||||
nextPageParameters: newNextPageParameters,
|
||||
hasBucket: newHasBucket,
|
||||
} = convertOnlineDriveData(documentsData.data, breadcrumbs, bucket)
|
||||
setOnlineDriveFileList([...onlineDriveFileList, ...newFileList])
|
||||
isTruncated.current = newIsTruncated
|
||||
currentNextPageParametersRef.current = newNextPageParameters
|
||||
setHasBucket(newHasBucket)
|
||||
setIsLoading(false)
|
||||
isLoadingRef.current = false
|
||||
},
|
||||
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
|
||||
toast.error(error.error)
|
||||
setIsLoading(false)
|
||||
isLoadingRef.current = false
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [dataSourceStore, datasourceNodeRunURL, breadcrumbs])
|
||||
|
||||
useEffect(() => {
|
||||
if (!currentCredentialId) return
|
||||
if (isInitialMount) {
|
||||
// Only fetch files on initial mount if fileList is empty
|
||||
if (onlineDriveFileList.length === 0) getOnlineDriveFiles()
|
||||
setIsInitialMount(false)
|
||||
} else {
|
||||
getOnlineDriveFiles()
|
||||
}
|
||||
}, [nextPageParameters, prefix, bucket, currentCredentialId])
|
||||
|
||||
const filteredOnlineDriveFileList = useMemo(() => {
|
||||
if (keywords)
|
||||
return onlineDriveFileList.filter((file) =>
|
||||
file.name.toLowerCase().includes(keywords.toLowerCase()),
|
||||
)
|
||||
return onlineDriveFileList
|
||||
}, [onlineDriveFileList, keywords])
|
||||
|
||||
const updateKeywords = useCallback(
|
||||
(keywords: string) => {
|
||||
const { setKeywords } = dataSourceStore.getState()
|
||||
setKeywords(keywords)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const resetKeywords = useCallback(() => {
|
||||
const { setKeywords } = dataSourceStore.getState()
|
||||
|
||||
setKeywords('')
|
||||
}, [dataSourceStore])
|
||||
|
||||
const handleSelectFile = useCallback(
|
||||
(file: OnlineDriveFile) => {
|
||||
const { selectedFileIds, setSelectedFileIds } = dataSourceStore.getState()
|
||||
if (file.type === OnlineDriveFileType.bucket) return
|
||||
const newSelectedFileList = produce(selectedFileIds, (draft) => {
|
||||
if (draft.includes(file.id)) {
|
||||
const index = draft.indexOf(file.id)
|
||||
draft.splice(index, 1)
|
||||
} else {
|
||||
if (!supportBatchUpload && draft.length >= 1) return
|
||||
draft.push(file.id)
|
||||
}
|
||||
})
|
||||
setSelectedFileIds(newSelectedFileList)
|
||||
},
|
||||
[dataSourceStore, supportBatchUpload],
|
||||
)
|
||||
|
||||
const handleOpenFolder = useCallback(
|
||||
(file: OnlineDriveFile) => {
|
||||
const {
|
||||
breadcrumbs,
|
||||
prefix,
|
||||
setBreadcrumbs,
|
||||
setPrefix,
|
||||
setBucket,
|
||||
setOnlineDriveFileList,
|
||||
setSelectedFileIds,
|
||||
} = dataSourceStore.getState()
|
||||
if (file.type === OnlineDriveFileType.file) return
|
||||
setOnlineDriveFileList([])
|
||||
if (file.type === OnlineDriveFileType.bucket) {
|
||||
setBucket(file.name)
|
||||
} else {
|
||||
setSelectedFileIds([])
|
||||
const newBreadcrumbs = produce(breadcrumbs, (draft) => {
|
||||
draft.push(file.name)
|
||||
})
|
||||
const newPrefix = produce(prefix, (draft) => {
|
||||
draft.push(file.id)
|
||||
})
|
||||
setBreadcrumbs(newBreadcrumbs)
|
||||
setPrefix(newPrefix)
|
||||
}
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const handleSetting = useCallback(() => {
|
||||
setSettingsDestination('data-source')
|
||||
}, [setSettingsDestination])
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-y-2">
|
||||
<Header
|
||||
docTitle="Docs"
|
||||
docLink={docLink('/use-dify/knowledge/knowledge-pipeline/authorize-data-source')}
|
||||
onClickConfiguration={handleSetting}
|
||||
pluginName={nodeData.datasource_label}
|
||||
currentCredentialId={currentCredentialId}
|
||||
onCredentialChange={onCredentialChange}
|
||||
credentials={dataSourceAuth?.result || []}
|
||||
/>
|
||||
<FileList
|
||||
fileList={filteredOnlineDriveFileList}
|
||||
selectedFileIds={selectedFileIds}
|
||||
breadcrumbs={breadcrumbs}
|
||||
keywords={keywords}
|
||||
bucket={bucket}
|
||||
resetKeywords={resetKeywords}
|
||||
updateKeywords={updateKeywords}
|
||||
searchResultsLength={filteredOnlineDriveFileList.length}
|
||||
handleSelectFile={handleSelectFile}
|
||||
handleOpenFolder={handleOpenFolder}
|
||||
isInPipeline={isInPipeline}
|
||||
isLoading={isLoading}
|
||||
supportBatchUpload={supportBatchUpload}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default OnlineDrive
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import type { OnlineDriveData } from '@/types/pipeline'
|
||||
import { OnlineDriveFileType } from '@/models/pipeline'
|
||||
|
||||
const isFile = (type: 'file' | 'folder'): boolean => {
|
||||
return type === 'file'
|
||||
}
|
||||
|
||||
export const isBucketListInitiation = (
|
||||
data: OnlineDriveData[],
|
||||
prefix: string[],
|
||||
bucket: string,
|
||||
): boolean => {
|
||||
if (bucket || prefix.length > 0) return false
|
||||
const hasBucket = data.every((item) => !!item.bucket)
|
||||
return (
|
||||
hasBucket &&
|
||||
(data.length > 1 || (data.length === 1 && !!data[0]!.bucket && data[0]!.files.length === 0))
|
||||
)
|
||||
}
|
||||
|
||||
export const convertOnlineDriveData = (
|
||||
data: OnlineDriveData[],
|
||||
prefix: string[],
|
||||
bucket: string,
|
||||
): {
|
||||
fileList: OnlineDriveFile[]
|
||||
isTruncated: boolean
|
||||
nextPageParameters: Record<string, unknown>
|
||||
hasBucket: boolean
|
||||
} => {
|
||||
const fileList: OnlineDriveFile[] = []
|
||||
let isTruncated = false
|
||||
let nextPageParameters: Record<string, unknown> = {}
|
||||
let hasBucket = false
|
||||
|
||||
if (data.length === 0) return { fileList, isTruncated, nextPageParameters, hasBucket }
|
||||
|
||||
if (isBucketListInitiation(data, prefix, bucket)) {
|
||||
data.forEach((item) => {
|
||||
fileList.push({
|
||||
id: item.bucket,
|
||||
name: item.bucket,
|
||||
type: OnlineDriveFileType.bucket,
|
||||
})
|
||||
})
|
||||
hasBucket = true
|
||||
} else {
|
||||
data[0]!.files.forEach((file) => {
|
||||
const { id, name, size, type } = file
|
||||
const isFileType = isFile(type)
|
||||
fileList.push({
|
||||
id,
|
||||
name,
|
||||
size: isFileType ? size : undefined,
|
||||
type: isFileType ? OnlineDriveFileType.file : OnlineDriveFileType.folder,
|
||||
})
|
||||
})
|
||||
isTruncated = data[0]!.is_truncated ?? false
|
||||
nextPageParameters = data[0]!.next_page_parameters ?? {}
|
||||
hasBucket = !!data[0]!.bucket
|
||||
}
|
||||
return { fileList, isTruncated, nextPageParameters, hasBucket }
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
import type { CommonShape } from './slices/common'
|
||||
import type { LocalFileSliceShape } from './slices/local-file'
|
||||
import type { OnlineDocumentSliceShape } from './slices/online-document'
|
||||
import type { OnlineDriveSliceShape } from './slices/online-drive'
|
||||
import type { WebsiteCrawlSliceShape } from './slices/website-crawl'
|
||||
import { use } from 'react'
|
||||
import { createStore, useStore } from 'zustand'
|
||||
import { DataSourceContext } from './provider'
|
||||
import { createCommonSlice } from './slices/common'
|
||||
import { createLocalFileSlice } from './slices/local-file'
|
||||
import { createOnlineDocumentSlice } from './slices/online-document'
|
||||
import { createOnlineDriveSlice } from './slices/online-drive'
|
||||
import { createWebsiteCrawlSlice } from './slices/website-crawl'
|
||||
|
||||
export type DataSourceShape = CommonShape &
|
||||
LocalFileSliceShape &
|
||||
OnlineDocumentSliceShape &
|
||||
WebsiteCrawlSliceShape &
|
||||
OnlineDriveSliceShape
|
||||
|
||||
export const createDataSourceStore = () => {
|
||||
return createStore<DataSourceShape>((...args) => ({
|
||||
...createCommonSlice(...args),
|
||||
...createLocalFileSlice(...args),
|
||||
...createOnlineDocumentSlice(...args),
|
||||
...createWebsiteCrawlSlice(...args),
|
||||
...createOnlineDriveSlice(...args),
|
||||
}))
|
||||
}
|
||||
|
||||
export const useDataSourceStoreWithSelector = <T>(selector: (state: DataSourceShape) => T): T => {
|
||||
const store = use(DataSourceContext)
|
||||
if (!store) throw new Error('Missing DataSourceContext.Provider in the tree')
|
||||
|
||||
return useStore(store, selector)
|
||||
}
|
||||
|
||||
export const useDataSourceStore = () => {
|
||||
const store = use(DataSourceContext)
|
||||
if (!store) throw new Error('Missing DataSourceContext.Provider in the tree')
|
||||
|
||||
return store
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
import { useRefWithInit } from '@langgenius/dify-ui/use-ref-with-init'
|
||||
import { createContext } from 'react'
|
||||
import { createDataSourceStore } from './'
|
||||
|
||||
type DataSourceStoreApi = ReturnType<typeof createDataSourceStore>
|
||||
|
||||
type DataSourceContextType = DataSourceStoreApi | null
|
||||
|
||||
export const DataSourceContext = createContext<DataSourceContextType>(null)
|
||||
|
||||
type DataSourceProviderProps = {
|
||||
children: React.ReactNode
|
||||
}
|
||||
|
||||
const DataSourceProvider = ({ children }: DataSourceProviderProps) => {
|
||||
const storeRef = useRefWithInit(createDataSourceStore)
|
||||
|
||||
return (
|
||||
<DataSourceContext.Provider value={storeRef.current}>{children}</DataSourceContext.Provider>
|
||||
)
|
||||
}
|
||||
|
||||
export default DataSourceProvider
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
import type { StateCreator } from 'zustand'
|
||||
|
||||
export type CommonShape = {
|
||||
currentNodeIdRef: React.RefObject<string>
|
||||
currentCredentialId: string
|
||||
setCurrentCredentialId: (credentialId: string) => void
|
||||
currentCredentialIdRef: React.RefObject<string>
|
||||
}
|
||||
|
||||
export const createCommonSlice: StateCreator<CommonShape> = (set) => {
|
||||
return {
|
||||
currentNodeIdRef: { current: '' },
|
||||
currentCredentialId: '',
|
||||
setCurrentCredentialId: (credentialId: string) => {
|
||||
set({ currentCredentialId: credentialId })
|
||||
},
|
||||
currentCredentialIdRef: { current: '' },
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
import type { StateCreator } from 'zustand'
|
||||
import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets'
|
||||
|
||||
export type LocalFileSliceShape = {
|
||||
localFileList: FileItem[]
|
||||
setLocalFileList: (fileList: FileItem[]) => void
|
||||
currentLocalFile: File | undefined
|
||||
setCurrentLocalFile: (file: File | undefined) => void
|
||||
previewLocalFileRef: React.RefObject<DocumentItem | undefined>
|
||||
}
|
||||
|
||||
export const createLocalFileSlice: StateCreator<LocalFileSliceShape> = (set, get) => {
|
||||
return {
|
||||
localFileList: [],
|
||||
setLocalFileList: (fileList: FileItem[]) => {
|
||||
set(() => ({
|
||||
localFileList: fileList,
|
||||
}))
|
||||
const { previewLocalFileRef } = get()
|
||||
previewLocalFileRef.current = fileList[0]?.file as DocumentItem
|
||||
},
|
||||
currentLocalFile: undefined,
|
||||
setCurrentLocalFile: (file: File | undefined) =>
|
||||
set(() => ({
|
||||
currentLocalFile: file,
|
||||
})),
|
||||
previewLocalFileRef: { current: undefined },
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import type { StateCreator } from 'zustand'
|
||||
import type { DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
|
||||
export type OnlineDocumentSliceShape = {
|
||||
documentsData: DataSourceNotionWorkspace[]
|
||||
setDocumentsData: (documentData: DataSourceNotionWorkspace[]) => void
|
||||
searchValue: string
|
||||
setSearchValue: (searchValue: string) => void
|
||||
onlineDocuments: NotionPage[]
|
||||
setOnlineDocuments: (documents: NotionPage[]) => void
|
||||
currentDocument: NotionPage | undefined
|
||||
setCurrentDocument: (document: NotionPage | undefined) => void
|
||||
selectedPagesId: Set<string>
|
||||
setSelectedPagesId: (selectedPagesId: Set<string>) => void
|
||||
previewOnlineDocumentRef: React.RefObject<NotionPage | undefined>
|
||||
}
|
||||
|
||||
export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set, get) => {
|
||||
return {
|
||||
documentsData: [],
|
||||
setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) =>
|
||||
set(() => ({
|
||||
documentsData,
|
||||
})),
|
||||
searchValue: '',
|
||||
setSearchValue: (searchValue: string) =>
|
||||
set(() => ({
|
||||
searchValue,
|
||||
})),
|
||||
onlineDocuments: [],
|
||||
setOnlineDocuments: (documents: NotionPage[]) => {
|
||||
set(() => ({
|
||||
onlineDocuments: documents,
|
||||
}))
|
||||
const { previewOnlineDocumentRef } = get()
|
||||
previewOnlineDocumentRef.current = documents[0]
|
||||
},
|
||||
currentDocument: undefined,
|
||||
setCurrentDocument: (document: NotionPage | undefined) =>
|
||||
set(() => ({
|
||||
currentDocument: document,
|
||||
})),
|
||||
selectedPagesId: new Set(),
|
||||
setSelectedPagesId: (selectedPagesId: Set<string>) =>
|
||||
set(() => ({
|
||||
selectedPagesId,
|
||||
})),
|
||||
previewOnlineDocumentRef: { current: undefined },
|
||||
}
|
||||
}
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
import type { StateCreator } from 'zustand'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
|
||||
export type OnlineDriveSliceShape = {
|
||||
breadcrumbs: string[]
|
||||
setBreadcrumbs: (breadcrumbs: string[]) => void
|
||||
prefix: string[]
|
||||
setPrefix: (prefix: string[]) => void
|
||||
keywords: string
|
||||
setKeywords: (keywords: string) => void
|
||||
selectedFileIds: string[]
|
||||
setSelectedFileIds: (selectedFileIds: string[]) => void
|
||||
onlineDriveFileList: OnlineDriveFile[]
|
||||
setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => void
|
||||
bucket: string
|
||||
setBucket: (bucket: string) => void
|
||||
nextPageParameters: Record<string, unknown>
|
||||
currentNextPageParametersRef: React.RefObject<Record<string, unknown>>
|
||||
setNextPageParameters: (nextPageParameters: Record<string, unknown>) => void
|
||||
isTruncated: React.RefObject<boolean>
|
||||
previewOnlineDriveFileRef: React.RefObject<OnlineDriveFile | undefined>
|
||||
hasBucket: boolean
|
||||
setHasBucket: (hasBucket: boolean) => void
|
||||
}
|
||||
|
||||
export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set, get) => {
|
||||
return {
|
||||
breadcrumbs: [],
|
||||
setBreadcrumbs: (breadcrumbs: string[]) =>
|
||||
set(() => ({
|
||||
breadcrumbs,
|
||||
})),
|
||||
prefix: [],
|
||||
setPrefix: (prefix: string[]) =>
|
||||
set(() => ({
|
||||
prefix,
|
||||
})),
|
||||
keywords: '',
|
||||
setKeywords: (keywords: string) =>
|
||||
set(() => ({
|
||||
keywords,
|
||||
})),
|
||||
selectedFileIds: [],
|
||||
setSelectedFileIds: (selectedFileIds: string[]) => {
|
||||
set(() => ({
|
||||
selectedFileIds,
|
||||
}))
|
||||
const id = selectedFileIds[0]
|
||||
const { onlineDriveFileList, previewOnlineDriveFileRef } = get()
|
||||
previewOnlineDriveFileRef.current = onlineDriveFileList.find((file) => file.id === id)
|
||||
},
|
||||
onlineDriveFileList: [],
|
||||
setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) =>
|
||||
set(() => ({
|
||||
onlineDriveFileList,
|
||||
})),
|
||||
bucket: '',
|
||||
setBucket: (bucket: string) =>
|
||||
set(() => ({
|
||||
bucket,
|
||||
})),
|
||||
nextPageParameters: {},
|
||||
currentNextPageParametersRef: { current: {} },
|
||||
setNextPageParameters: (nextPageParameters: Record<string, unknown>) =>
|
||||
set(() => ({
|
||||
nextPageParameters,
|
||||
})),
|
||||
isTruncated: { current: false },
|
||||
previewOnlineDriveFileRef: { current: undefined },
|
||||
hasBucket: false,
|
||||
setHasBucket: (hasBucket: boolean) =>
|
||||
set(() => ({
|
||||
hasBucket,
|
||||
})),
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
import type { StateCreator } from 'zustand'
|
||||
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
|
||||
export type WebsiteCrawlSliceShape = {
|
||||
websitePages: CrawlResultItem[]
|
||||
setWebsitePages: (pages: CrawlResultItem[]) => void
|
||||
currentWebsite: CrawlResultItem | undefined
|
||||
setCurrentWebsite: (website: CrawlResultItem | undefined) => void
|
||||
crawlResult: CrawlResult | undefined
|
||||
setCrawlResult: (result: CrawlResult | undefined) => void
|
||||
step: CrawlStep
|
||||
setStep: (step: CrawlStep) => void
|
||||
previewIndex: number
|
||||
setPreviewIndex: (index: number) => void
|
||||
previewWebsitePageRef: React.RefObject<CrawlResultItem | undefined>
|
||||
}
|
||||
|
||||
export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (set, get) => {
|
||||
return {
|
||||
websitePages: [],
|
||||
setWebsitePages: (pages: CrawlResultItem[]) => {
|
||||
set(() => ({
|
||||
websitePages: pages,
|
||||
}))
|
||||
const { previewWebsitePageRef } = get()
|
||||
previewWebsitePageRef.current = pages[0]
|
||||
},
|
||||
currentWebsite: undefined,
|
||||
setCurrentWebsite: (website: CrawlResultItem | undefined) =>
|
||||
set(() => ({
|
||||
currentWebsite: website,
|
||||
})),
|
||||
crawlResult: undefined,
|
||||
setCrawlResult: (result: CrawlResult | undefined) =>
|
||||
set(() => ({
|
||||
crawlResult: result,
|
||||
})),
|
||||
step: CrawlStep.init,
|
||||
setStep: (step: CrawlStep) =>
|
||||
set(() => ({
|
||||
step,
|
||||
})),
|
||||
previewIndex: -1,
|
||||
setPreviewIndex: (index: number) =>
|
||||
set(() => ({
|
||||
previewIndex: index,
|
||||
})),
|
||||
previewWebsitePageRef: { current: undefined },
|
||||
}
|
||||
}
|
||||
+310
@@ -0,0 +1,310 @@
|
||||
import type { DatasourceCredentialResponse } from '@dify/contracts/api/console/auth/types.gen'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
import type {
|
||||
DataSourceNodeCompletedResponse,
|
||||
DataSourceNodeErrorResponse,
|
||||
DataSourceNodeProcessingResponse,
|
||||
} from '@/types/pipeline'
|
||||
import { act, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { BlockEnum } from '@/app/components/workflow/types'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import { createConsoleQueryClient, renderWithConsoleQuery } from '@/test/console/query-data'
|
||||
import { createNuqsTestRenderer } from '@/test/nuqs-testing'
|
||||
import { createPipelineVariable } from '@/test/pipeline-variables'
|
||||
import { createDataSourceStore } from '../../store'
|
||||
import { DataSourceContext } from '../../store/provider'
|
||||
import WebsiteCrawl from '../index'
|
||||
|
||||
const { ssePost, parameters } = vi.hoisted(() => ({ ssePost: vi.fn(), parameters: vi.fn() }))
|
||||
vi.mock('@/service/base', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('@/service/base')>()),
|
||||
ssePost,
|
||||
}))
|
||||
vi.mock('@/context/dataset-detail', () => ({
|
||||
useDatasetDetailContextWithSelector: (
|
||||
select: (state: { dataset: { pipeline_id: string } }) => unknown,
|
||||
) => select({ dataset: { pipeline_id: 'pipeline-123' } }),
|
||||
}))
|
||||
const onUrlUpdate = vi.fn()
|
||||
const render = createNuqsTestRenderer(renderWithConsoleQuery, () => ({ onUrlUpdate }))
|
||||
const nodeData: DataSourceNodeType = {
|
||||
type: BlockEnum.DataSource,
|
||||
title: 'Crawler',
|
||||
desc: '',
|
||||
provider_type: DatasourceType.websiteCrawl,
|
||||
plugin_id: 'plugin-123',
|
||||
provider_name: 'crawler',
|
||||
datasource_name: 'crawler',
|
||||
datasource_label: 'Crawler',
|
||||
datasource_parameters: {},
|
||||
datasource_configurations: {},
|
||||
}
|
||||
const credentials: DatasourceCredentialResponse[] = [
|
||||
{
|
||||
id: 'credential-1',
|
||||
name: 'First credential',
|
||||
avatar_url: '',
|
||||
type: 'oauth2',
|
||||
is_default: false,
|
||||
credential: {},
|
||||
},
|
||||
{
|
||||
id: 'credential-2',
|
||||
name: 'Second credential',
|
||||
avatar_url: '',
|
||||
type: 'oauth2',
|
||||
is_default: false,
|
||||
credential: {},
|
||||
},
|
||||
]
|
||||
const pages: CrawlResultItem[] = [
|
||||
{
|
||||
title: 'First page',
|
||||
source_url: 'https://example.com/1',
|
||||
markdown: '# First',
|
||||
description: 'First',
|
||||
},
|
||||
{
|
||||
title: 'Second page',
|
||||
source_url: 'https://example.com/2',
|
||||
markdown: '# Second',
|
||||
description: 'Second',
|
||||
},
|
||||
]
|
||||
function setup({
|
||||
credential = 'credential-1',
|
||||
single = false,
|
||||
draft = false,
|
||||
loading = false,
|
||||
} = {}) {
|
||||
parameters.mockImplementation(() =>
|
||||
loading
|
||||
? new Promise(() => {})
|
||||
: Promise.resolve({
|
||||
variables: [
|
||||
createPipelineVariable({
|
||||
variable: 'url',
|
||||
label: 'URL',
|
||||
default_value: 'https://example.com',
|
||||
}),
|
||||
],
|
||||
}),
|
||||
)
|
||||
const store = createDataSourceStore()
|
||||
store.setState({ currentCredentialId: credential })
|
||||
const queryClient = createConsoleQueryClient()
|
||||
queryClient.setQueryData(
|
||||
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
|
||||
input: { params: { provider_id: 'plugin-123/crawler' } },
|
||||
}).queryKey,
|
||||
{ result: credential ? credentials : [] },
|
||||
)
|
||||
const workflows = consoleQuery.rag.pipelines.byPipelineId.workflows
|
||||
const options = (
|
||||
draft ? workflows.draft : workflows.published
|
||||
).preProcessing.parameters.get.queryOptions({
|
||||
input: { params: { pipeline_id: 'pipeline-123' }, query: { node_id: 'node-1' } },
|
||||
})
|
||||
if (loading)
|
||||
queryClient.setQueryDefaults(options.queryKey, { queryFn: () => new Promise(() => {}) })
|
||||
else
|
||||
queryClient.setQueryData(options.queryKey, {
|
||||
variables: [
|
||||
createPipelineVariable({
|
||||
variable: 'url',
|
||||
label: 'URL',
|
||||
default_value: 'https://example.com',
|
||||
}),
|
||||
],
|
||||
})
|
||||
const onCredentialChange = vi.fn((id: string) => {
|
||||
store.setState({
|
||||
currentCredentialId: id,
|
||||
step: CrawlStep.init,
|
||||
websitePages: [],
|
||||
crawlResult: undefined,
|
||||
})
|
||||
})
|
||||
render(
|
||||
<DataSourceContext value={store}>
|
||||
<WebsiteCrawl
|
||||
nodeId="node-1"
|
||||
nodeData={nodeData}
|
||||
onCredentialChange={onCredentialChange}
|
||||
supportBatchUpload={!single}
|
||||
isInPipeline={draft}
|
||||
/>
|
||||
</DataSourceContext>,
|
||||
{ queryClient },
|
||||
)
|
||||
return { store, onCredentialChange }
|
||||
}
|
||||
function callbacks(): {
|
||||
onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => void
|
||||
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => void
|
||||
onDataSourceNodeError: (data: DataSourceNodeErrorResponse) => void
|
||||
} {
|
||||
return ssePost.mock.lastCall![2]
|
||||
}
|
||||
const runButton = () => screen.getByRole('button', { name: 'datasetCreation.stepOne.website.run' })
|
||||
function complete() {
|
||||
act(() =>
|
||||
callbacks().onDataSourceNodeCompleted({
|
||||
event: 'datasource_completed',
|
||||
time_consuming: 2.5,
|
||||
data: pages,
|
||||
}),
|
||||
)
|
||||
}
|
||||
beforeEach(() => vi.clearAllMocks())
|
||||
|
||||
it.each([true, false])(
|
||||
'runs the draft=%s workflow, displays streamed progress and selects completed pages',
|
||||
async (draft) => {
|
||||
const user = userEvent.setup()
|
||||
const { store } = setup({ draft })
|
||||
await waitFor(() => expect(runButton()).toBeEnabled())
|
||||
const input = screen.getByRole('textbox', { name: /URL/ })
|
||||
await user.clear(input)
|
||||
await user.type(input, 'https://example.com/docs')
|
||||
await waitFor(() => expect(runButton()).toBeEnabled())
|
||||
await user.click(runButton())
|
||||
await waitFor(() =>
|
||||
expect(ssePost).toHaveBeenCalledExactlyOnceWith(
|
||||
`/rag/pipelines/pipeline-123/workflows/${draft ? 'draft' : 'published'}/datasource/nodes/node-1/run`,
|
||||
{
|
||||
body: {
|
||||
inputs: { url: 'https://example.com/docs' },
|
||||
credential_id: 'credential-1',
|
||||
datasource_type: DatasourceType.websiteCrawl,
|
||||
response_mode: 'streaming',
|
||||
},
|
||||
},
|
||||
expect.any(Object),
|
||||
),
|
||||
)
|
||||
act(() =>
|
||||
callbacks().onDataSourceNodeProcessing({
|
||||
event: 'datasource_processing',
|
||||
total: 2,
|
||||
completed: 1,
|
||||
}),
|
||||
)
|
||||
expect(screen.getByText(/totalPageScraped/)).toHaveTextContent('1/2')
|
||||
await user.click(
|
||||
screen.getByRole('button', { name: 'datasetCreation.stepOne.website.running' }),
|
||||
)
|
||||
expect(ssePost).toHaveBeenCalledOnce()
|
||||
complete()
|
||||
expect(store.getState().step).toBe(CrawlStep.finished)
|
||||
expect(store.getState().websitePages).toEqual(pages)
|
||||
expect(screen.getByRole('checkbox', { name: /First page/ })).toBeChecked()
|
||||
expect(screen.getByRole('checkbox', { name: /Second page/ })).toBeChecked()
|
||||
expect(
|
||||
screen.queryAllByRole('button', { name: 'datasetCreation.stepOne.website.preview' }),
|
||||
).toHaveLength(draft ? 0 : 2)
|
||||
},
|
||||
)
|
||||
|
||||
it.each([{ credential: '' }, { loading: true }])(
|
||||
'prevents crawling when credentials or parameters are unavailable: %j',
|
||||
async (options) => {
|
||||
const user = userEvent.setup()
|
||||
setup(options)
|
||||
expect(runButton()).toBeDisabled()
|
||||
await user.click(runButton())
|
||||
expect(ssePost).not.toHaveBeenCalled()
|
||||
},
|
||||
)
|
||||
|
||||
it('keeps only one page selected in single-upload mode', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { store } = setup({ single: true })
|
||||
await waitFor(() => expect(runButton()).toBeEnabled())
|
||||
await user.click(runButton())
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
complete()
|
||||
expect(store.getState().websitePages).toEqual([pages[0]])
|
||||
await user.click(screen.getByRole('radio', { name: /Second page/ }))
|
||||
expect(store.getState().websitePages).toEqual([pages[1]])
|
||||
expect(screen.getByRole('radio', { name: /First page/ })).not.toBeChecked()
|
||||
})
|
||||
|
||||
it('deselects a page and previews it without changing the remaining selection', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { store } = setup()
|
||||
await waitFor(() => expect(runButton()).toBeEnabled())
|
||||
await user.click(runButton())
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
complete()
|
||||
await user.click(screen.getByRole('checkbox', { name: /Second page/ }))
|
||||
await user.click(
|
||||
screen.getAllByRole('button', { name: 'datasetCreation.stepOne.website.preview' })[1]!,
|
||||
)
|
||||
expect(store.getState().websitePages).toEqual([pages[0]])
|
||||
expect(store.getState().currentWebsite).toEqual(pages[1])
|
||||
expect(store.getState().previewIndex).toBe(1)
|
||||
})
|
||||
|
||||
it.each(['Crawler unavailable', ''])(
|
||||
'shows a stream error and clears it after a successful retry: %s',
|
||||
async (message) => {
|
||||
const user = userEvent.setup()
|
||||
const { store } = setup()
|
||||
await waitFor(() => expect(runButton()).toBeEnabled())
|
||||
await user.click(runButton())
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
act(() => callbacks().onDataSourceNodeError({ event: 'datasource_error', error: message }))
|
||||
const error = message || 'datasetCreation.stepOne.website.unknownError'
|
||||
expect(screen.getByText(error)).toBeInTheDocument()
|
||||
expect(store.getState().websitePages).toEqual([])
|
||||
await waitFor(() => expect(runButton()).toBeEnabled())
|
||||
await user.click(runButton())
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledTimes(2))
|
||||
complete()
|
||||
expect(screen.queryByText(error)).not.toBeInTheDocument()
|
||||
expect(store.getState().websitePages).toEqual(pages)
|
||||
},
|
||||
)
|
||||
|
||||
it('uses the changed credential on the next run and opens configuration', async () => {
|
||||
const user = userEvent.setup()
|
||||
const { onCredentialChange } = setup()
|
||||
await waitFor(() => expect(runButton()).toBeEnabled())
|
||||
await user.click(runButton())
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledOnce())
|
||||
act(() =>
|
||||
callbacks().onDataSourceNodeError({
|
||||
event: 'datasource_error',
|
||||
error: 'Old credential failed',
|
||||
}),
|
||||
)
|
||||
await user.click(screen.getByRole('button', { name: /First credential/ }))
|
||||
await user.click(screen.getByText('Second credential'))
|
||||
expect(onCredentialChange).toHaveBeenCalledExactlyOnceWith('credential-2')
|
||||
await waitFor(() => expect(runButton()).toBeEnabled())
|
||||
await user.click(runButton())
|
||||
await waitFor(() => expect(ssePost).toHaveBeenCalledTimes(2))
|
||||
expect(ssePost.mock.lastCall![1].body.credential_id).toBe('credential-2')
|
||||
expect(screen.queryByText('Old credential failed')).not.toBeInTheDocument()
|
||||
await user.click(screen.getByRole('button', { name: /^datasetPipeline\.configurationTip/ }))
|
||||
await waitFor(() =>
|
||||
expect(onUrlUpdate.mock.lastCall![0].searchParams.get('settings')).toBe('data-source'),
|
||||
)
|
||||
})
|
||||
|
||||
vi.mock('@/service/console', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@/service/console')>()
|
||||
const { withConsoleResponses } = await import('@/test/console/service-responses')
|
||||
return {
|
||||
...actual,
|
||||
consoleQuery: withConsoleResponses(actual.consoleQuery, {
|
||||
'rag.pipelines.byPipelineId.workflows.published.preProcessing.parameters.get': parameters,
|
||||
'rag.pipelines.byPipelineId.workflows.draft.preProcessing.parameters.get': parameters,
|
||||
}),
|
||||
}
|
||||
})
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { useState } from 'react'
|
||||
import CrawledResult from '../crawled-result'
|
||||
|
||||
const pages: CrawlResultItem[] = ['First', 'Second', 'Third'].map((title, index) => ({
|
||||
title,
|
||||
source_url: `https://example.com/${index}`,
|
||||
markdown: title,
|
||||
description: title,
|
||||
}))
|
||||
function Harness({ multiple = true, onChange = vi.fn(), onPreview = vi.fn() }) {
|
||||
const [selected, setSelected] = useState([pages[0]!])
|
||||
return (
|
||||
<CrawledResult
|
||||
list={pages}
|
||||
checkedList={selected}
|
||||
usedTime={12.3}
|
||||
isMultipleChoice={multiple}
|
||||
showPreview
|
||||
onPreview={onPreview}
|
||||
onSelectedChange={(next) => {
|
||||
setSelected(next)
|
||||
onChange(next)
|
||||
}}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
it('adds and removes individual pages and supports selecting and resetting all', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onChange = vi.fn()
|
||||
render(<Harness onChange={onChange} />)
|
||||
await user.click(screen.getByRole('checkbox', { name: 'Second https://example.com/1' }))
|
||||
expect(onChange).toHaveBeenLastCalledWith([pages[0], pages[1]])
|
||||
await user.click(screen.getByRole('checkbox', { name: 'First https://example.com/0' }))
|
||||
expect(onChange).toHaveBeenLastCalledWith([pages[1]])
|
||||
await user.click(
|
||||
screen.getByRole('checkbox', { name: 'datasetCreation.stepOne.website.selectAll' }),
|
||||
)
|
||||
expect(onChange).toHaveBeenLastCalledWith(pages)
|
||||
await user.click(
|
||||
screen.getByRole('checkbox', { name: 'datasetCreation.stepOne.website.resetAll' }),
|
||||
)
|
||||
expect(onChange).toHaveBeenLastCalledWith([])
|
||||
})
|
||||
|
||||
it('replaces the selected page in single-choice mode and previews independently', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onChange = vi.fn()
|
||||
const onPreview = vi.fn()
|
||||
render(<Harness multiple={false} onChange={onChange} onPreview={onPreview} />)
|
||||
expect(screen.queryByRole('checkbox')).not.toBeInTheDocument()
|
||||
await user.click(screen.getByRole('radio', { name: 'Second https://example.com/1' }))
|
||||
expect(onChange).toHaveBeenCalledWith([pages[1]])
|
||||
await user.click(
|
||||
screen.getAllByRole('button', { name: 'datasetCreation.stepOne.website.preview' })[2]!,
|
||||
)
|
||||
expect(onPreview).toHaveBeenCalledWith(pages[2], 2)
|
||||
expect(onChange).toHaveBeenCalledOnce()
|
||||
})
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
'use client'
|
||||
import { Checkbox } from '@langgenius/dify-ui/checkbox'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { Infotip, InfotipContent, InfotipTrigger } from '@langgenius/dify-ui/infotip'
|
||||
import { useId } from 'react'
|
||||
|
||||
type CheckboxWithLabelProps = {
|
||||
className?: string
|
||||
isChecked: boolean
|
||||
onChange: (isChecked: boolean) => void
|
||||
label: string
|
||||
labelClassName?: string
|
||||
tooltip?: string
|
||||
}
|
||||
|
||||
export default function CheckboxWithLabel({
|
||||
className = '',
|
||||
isChecked,
|
||||
onChange,
|
||||
label,
|
||||
labelClassName,
|
||||
tooltip,
|
||||
}: CheckboxWithLabelProps) {
|
||||
const labelId = useId()
|
||||
|
||||
return (
|
||||
<div className={cn('flex items-center', className)}>
|
||||
<label className="flex min-w-0 cursor-pointer items-center">
|
||||
<Checkbox checked={isChecked} onCheckedChange={(checked) => onChange(checked)} />
|
||||
<span
|
||||
id={labelId}
|
||||
className={cn(
|
||||
'ml-2 min-w-0 text-left system-sm-medium text-text-secondary',
|
||||
labelClassName,
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
</span>
|
||||
</label>
|
||||
<div className="ml-1 flex min-w-0 items-center">
|
||||
{tooltip && (
|
||||
<Infotip>
|
||||
<InfotipTrigger aria-labelledby={labelId} />
|
||||
<InfotipContent aria-labelledby={labelId} className="w-50">
|
||||
{tooltip}
|
||||
</InfotipContent>
|
||||
</Infotip>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
'use client'
|
||||
import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import { Checkbox } from '@langgenius/dify-ui/checkbox'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { Radio } from '@langgenius/dify-ui/radio-group'
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type CrawledResultItemProps = {
|
||||
payload: CrawlResultItemType
|
||||
isChecked: boolean
|
||||
onCheckChange: (checked: boolean) => void
|
||||
isPreview: boolean
|
||||
showPreview: boolean
|
||||
onPreview: () => void
|
||||
isMultipleChoice?: boolean
|
||||
}
|
||||
|
||||
const CrawledResultItem = ({
|
||||
payload,
|
||||
isChecked,
|
||||
onCheckChange,
|
||||
isPreview,
|
||||
onPreview,
|
||||
showPreview,
|
||||
isMultipleChoice = true,
|
||||
}: CrawledResultItemProps) => {
|
||||
const { t } = useTranslation(['datasetCreation'])
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
'relative flex gap-x-2 rounded-lg p-2',
|
||||
isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover',
|
||||
)}
|
||||
>
|
||||
{isMultipleChoice ? (
|
||||
<label className="flex min-w-0 grow cursor-pointer gap-x-2">
|
||||
<Checkbox
|
||||
className="shrink-0"
|
||||
checked={isChecked}
|
||||
onCheckedChange={(checked) => onCheckChange(checked)}
|
||||
/>
|
||||
<div className="flex min-w-0 grow flex-col gap-y-0.5">
|
||||
<div className="truncate system-sm-medium text-text-secondary" title={payload.title}>
|
||||
{payload.title}
|
||||
</div>
|
||||
<div
|
||||
className="truncate system-xs-regular text-text-tertiary"
|
||||
title={payload.source_url}
|
||||
>
|
||||
{payload.source_url}
|
||||
</div>
|
||||
</div>
|
||||
</label>
|
||||
) : (
|
||||
<label className="flex min-w-0 grow cursor-pointer gap-x-2">
|
||||
<Radio className="shrink-0" value={payload.source_url} />
|
||||
<div className="flex min-w-0 grow flex-col gap-y-0.5">
|
||||
<div className="truncate system-sm-medium text-text-secondary" title={payload.title}>
|
||||
{payload.title}
|
||||
</div>
|
||||
<div
|
||||
className="truncate system-xs-regular text-text-tertiary"
|
||||
title={payload.source_url}
|
||||
>
|
||||
{payload.source_url}
|
||||
</div>
|
||||
</div>
|
||||
</label>
|
||||
)}
|
||||
{showPreview && (
|
||||
<Button
|
||||
size="small"
|
||||
onClick={onPreview}
|
||||
className="top-2 right-2 hidden px-1.5 system-xs-medium-uppercase group-hover:absolute group-hover:block"
|
||||
>
|
||||
{t(($) => $['stepOne.website.preview'], { ns: 'datasetCreation' })}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(CrawledResultItem)
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
'use client'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { RadioGroup } from '@langgenius/dify-ui/radio-group'
|
||||
import * as React from 'react'
|
||||
import { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import CheckboxWithLabel from './checkbox-with-label'
|
||||
import CrawledResultItem from './crawled-result-item'
|
||||
|
||||
const I18N_PREFIX = 'stepOne.website'
|
||||
|
||||
type CrawledResultProps = {
|
||||
className?: string
|
||||
previewIndex?: number
|
||||
list: CrawlResultItem[]
|
||||
checkedList: CrawlResultItem[]
|
||||
onSelectedChange: (selected: CrawlResultItem[]) => void
|
||||
onPreview?: (payload: CrawlResultItem, index: number) => void
|
||||
showPreview?: boolean
|
||||
usedTime: number
|
||||
isMultipleChoice?: boolean
|
||||
}
|
||||
|
||||
const CrawledResult = ({
|
||||
className = '',
|
||||
previewIndex,
|
||||
list,
|
||||
checkedList,
|
||||
onSelectedChange,
|
||||
usedTime,
|
||||
onPreview,
|
||||
showPreview = false,
|
||||
isMultipleChoice = true,
|
||||
}: CrawledResultProps) => {
|
||||
const { t } = useTranslation(['datasetCreation'])
|
||||
|
||||
const isCheckAll = checkedList.length === list.length
|
||||
|
||||
const handleCheckedAll = useCallback(() => {
|
||||
if (!isCheckAll) onSelectedChange(list)
|
||||
else onSelectedChange([])
|
||||
}, [isCheckAll, list, onSelectedChange])
|
||||
|
||||
const handleItemCheckChange = useCallback(
|
||||
(item: CrawlResultItem) => {
|
||||
return (checked: boolean) => {
|
||||
if (checked) {
|
||||
if (isMultipleChoice) onSelectedChange([...checkedList, item])
|
||||
else onSelectedChange([item])
|
||||
} else {
|
||||
onSelectedChange(
|
||||
checkedList.filter((checkedItem) => checkedItem.source_url !== item.source_url),
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
[checkedList, onSelectedChange, isMultipleChoice],
|
||||
)
|
||||
|
||||
const handlePreview = useCallback(
|
||||
(index: number) => {
|
||||
if (!onPreview) return
|
||||
onPreview(list[index]!, index)
|
||||
},
|
||||
[list, onPreview],
|
||||
)
|
||||
|
||||
const selectedSourceUrl = checkedList[0]?.source_url
|
||||
const handleRadioChange = useCallback(
|
||||
(sourceUrl: string) => {
|
||||
const selectedItem = list.find((item) => item.source_url === sourceUrl)
|
||||
if (selectedItem) onSelectedChange([selectedItem])
|
||||
},
|
||||
[list, onSelectedChange],
|
||||
)
|
||||
|
||||
const resultItems = list.map((item, index) => (
|
||||
<CrawledResultItem
|
||||
key={item.source_url}
|
||||
payload={item}
|
||||
isChecked={checkedList.some((checkedItem) => checkedItem.source_url === item.source_url)}
|
||||
onCheckChange={handleItemCheckChange(item)}
|
||||
isPreview={index === previewIndex}
|
||||
onPreview={handlePreview.bind(null, index)}
|
||||
showPreview={showPreview}
|
||||
isMultipleChoice={isMultipleChoice}
|
||||
/>
|
||||
))
|
||||
|
||||
return (
|
||||
<div className={cn('flex flex-col gap-y-2', className)}>
|
||||
<div className="pt-2 system-sm-medium text-text-primary">
|
||||
{t(($) => $[`${I18N_PREFIX}.scrapTimeInfo`], {
|
||||
ns: 'datasetCreation',
|
||||
total: list.length,
|
||||
time: usedTime.toFixed(1),
|
||||
})}
|
||||
</div>
|
||||
<div className="overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg">
|
||||
{isMultipleChoice && (
|
||||
<div className="flex items-center px-4 py-2">
|
||||
<CheckboxWithLabel
|
||||
isChecked={isCheckAll}
|
||||
onChange={handleCheckedAll}
|
||||
label={
|
||||
isCheckAll
|
||||
? t(($) => $[`${I18N_PREFIX}.resetAll`], { ns: 'datasetCreation' })
|
||||
: t(($) => $[`${I18N_PREFIX}.selectAll`], { ns: 'datasetCreation' })
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{isMultipleChoice ? (
|
||||
<div className="flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2">
|
||||
{resultItems}
|
||||
</div>
|
||||
) : (
|
||||
<RadioGroup
|
||||
aria-label={t(($) => $[`${I18N_PREFIX}.scrapTimeInfo`], {
|
||||
ns: 'datasetCreation',
|
||||
total: list.length,
|
||||
time: usedTime.toFixed(1),
|
||||
})}
|
||||
value={selectedSourceUrl}
|
||||
onValueChange={handleRadioChange}
|
||||
className="flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2"
|
||||
>
|
||||
{resultItems}
|
||||
</RadioGroup>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(CrawledResult)
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
'use client'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import * as React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
|
||||
type CrawlingProps = {
|
||||
className?: string
|
||||
crawledNum: number
|
||||
totalNum: number
|
||||
}
|
||||
|
||||
type BlockProps = {
|
||||
className?: string
|
||||
}
|
||||
|
||||
type ItemProps = {
|
||||
firstLineWidth: string
|
||||
secondLineWidth: string
|
||||
}
|
||||
|
||||
const Block = React.memo(({ className }: BlockProps) => {
|
||||
return <div className={cn('bg-text-quaternary opacity-20', className)} />
|
||||
})
|
||||
|
||||
const Item = React.memo(({ firstLineWidth, secondLineWidth }: ItemProps) => {
|
||||
return (
|
||||
<div className="flex gap-x-2 px-2 py-1.25">
|
||||
<div className="py-0.5">
|
||||
<Block className="size-4 rounded-sm" />
|
||||
</div>
|
||||
<div className="flex grow flex-col">
|
||||
<div className="flex h-5 w-full items-center">
|
||||
<Block className={cn('h-2.5 rounded-xs', firstLineWidth)} />
|
||||
</div>
|
||||
<div className="flex h-4.5 w-full items-center">
|
||||
<Block className={cn('h-1.5 rounded-xs', secondLineWidth)} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})
|
||||
|
||||
const Crawling = ({ className = '', crawledNum, totalNum }: CrawlingProps) => {
|
||||
const { t } = useTranslation(['datasetCreation'])
|
||||
|
||||
const itemsConfig = [
|
||||
{
|
||||
firstLineWidth: 'w-[35%]',
|
||||
secondLineWidth: 'w-[50%]',
|
||||
},
|
||||
{
|
||||
firstLineWidth: 'w-[40%]',
|
||||
secondLineWidth: 'w-[45%]',
|
||||
},
|
||||
{
|
||||
firstLineWidth: 'w-[30%]',
|
||||
secondLineWidth: 'w-[36%]',
|
||||
},
|
||||
]
|
||||
|
||||
return (
|
||||
<div className={cn('mt-2 flex flex-col gap-y-2 pt-2', className)}>
|
||||
<div className="system-sm-medium text-text-primary">
|
||||
{t(($) => $['stepOne.website.totalPageScraped'], { ns: 'datasetCreation' })} {crawledNum}/
|
||||
{totalNum}
|
||||
</div>
|
||||
<div className="overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg">
|
||||
<div className="flex items-center gap-x-2 px-4 py-2">
|
||||
<Block className="size-4 rounded-sm" />
|
||||
<Block className="h-2.5 w-14 rounded-xs" />
|
||||
</div>
|
||||
<div className="flex flex-col gap-px border-t border-divider-subtle bg-background-default-subtle p-2">
|
||||
{itemsConfig.map((item, index) => (
|
||||
<Item
|
||||
key={index}
|
||||
firstLineWidth={item.firstLineWidth}
|
||||
secondLineWidth={item.secondLineWidth}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Crawling)
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
import { Alert, AlertDescription, AlertTitle } from '@langgenius/dify-ui/alert'
|
||||
import * as React from 'react'
|
||||
|
||||
type ErrorMessageProps = {
|
||||
className?: string
|
||||
title: string
|
||||
errorMsg?: string
|
||||
}
|
||||
|
||||
const ErrorMessage = ({ className, title, errorMsg }: ErrorMessageProps) => {
|
||||
return (
|
||||
<Alert type="error" className={className}>
|
||||
<AlertTitle>{title}</AlertTitle>
|
||||
{errorMsg && <AlertDescription>{errorMsg}</AlertDescription>}
|
||||
</Alert>
|
||||
)
|
||||
}
|
||||
export default React.memo(ErrorMessage)
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
import { render, screen, waitFor } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { toast } from '@/app/notifications'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
import { createPipelineVariable } from '@/test/pipeline-variables'
|
||||
import Options from '../index'
|
||||
|
||||
vi.mock('@/app/notifications', () => ({ toast: { error: vi.fn() } }))
|
||||
const variables = [
|
||||
createPipelineVariable({ variable: 'url', label: 'URL', default_value: 'https://example.com' }),
|
||||
]
|
||||
const runName = 'datasetCreation.stepOne.website.run'
|
||||
|
||||
it('rejects an empty required field and runs with the corrected URL', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onSubmit = vi.fn()
|
||||
render(<Options variables={variables} step={CrawlStep.init} onSubmit={onSubmit} />)
|
||||
const input = screen.getByRole('textbox', { name: /URL/ })
|
||||
expect(input).toHaveValue('https://example.com')
|
||||
await user.clear(input)
|
||||
await user.click(screen.getByRole('button', { name: runName }))
|
||||
await waitFor(() => expect(toast.error).toHaveBeenCalledWith('"url" URL is required'))
|
||||
expect(onSubmit).not.toHaveBeenCalled()
|
||||
await user.type(input, 'https://example.com/docs')
|
||||
await user.click(screen.getByRole('button', { name: runName }))
|
||||
await waitFor(() =>
|
||||
expect(onSubmit).toHaveBeenCalledExactlyOnceWith({ url: 'https://example.com/docs' }),
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves edited options across crawling and reopens them for another run', async () => {
|
||||
const user = userEvent.setup()
|
||||
const props = { variables, onSubmit: vi.fn() }
|
||||
const { rerender } = render(<Options {...props} step={CrawlStep.init} />)
|
||||
const input = screen.getByRole('textbox', { name: /URL/ })
|
||||
await user.clear(input)
|
||||
await user.type(input, 'https://example.com/changed')
|
||||
rerender(<Options {...props} step={CrawlStep.running} />)
|
||||
expect(screen.queryByRole('textbox')).not.toBeInTheDocument()
|
||||
await user.click(screen.getByRole('button', { name: 'datasetCreation.stepOne.website.running' }))
|
||||
expect(props.onSubmit).not.toHaveBeenCalled()
|
||||
rerender(<Options {...props} step={CrawlStep.finished} />)
|
||||
expect(screen.queryByRole('textbox')).not.toBeInTheDocument()
|
||||
rerender(<Options {...props} step={CrawlStep.init} />)
|
||||
expect(screen.getByRole('textbox', { name: /URL/ })).toHaveValue('https://example.com/changed')
|
||||
})
|
||||
|
||||
it('does not start a crawl when the caller disables running', async () => {
|
||||
const user = userEvent.setup()
|
||||
const onSubmit = vi.fn()
|
||||
render(<Options variables={variables} step={CrawlStep.init} onSubmit={onSubmit} runDisabled />)
|
||||
await user.click(screen.getByRole('button', { name: runName }))
|
||||
expect(onSubmit).not.toHaveBeenCalled()
|
||||
})
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
import type { RAGPipelineVariables } from '@/models/pipeline'
|
||||
import { Button } from '@langgenius/dify-ui/button'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import { RiPlayLargeLine } from '@remixicon/react'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import { useEffect, useId, useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useAppForm } from '@/app/components/base/form'
|
||||
import BaseField from '@/app/components/base/form/form-scenarios/base/field'
|
||||
import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
|
||||
import {
|
||||
useConfigurations,
|
||||
useInitialData,
|
||||
} from '@/app/components/rag-pipeline/hooks/use-input-fields'
|
||||
import { toast } from '@/app/notifications'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
|
||||
const I18N_PREFIX = 'stepOne.website'
|
||||
|
||||
type OptionsProps = {
|
||||
variables: RAGPipelineVariables
|
||||
step: CrawlStep
|
||||
runDisabled?: boolean
|
||||
onSubmit: (data: Record<string, any>) => void
|
||||
}
|
||||
|
||||
const Options = ({ variables, step, runDisabled, onSubmit }: OptionsProps) => {
|
||||
const { t } = useTranslation(['datasetCreation'])
|
||||
const runButtonLabelId = useId()
|
||||
const initialData = useInitialData(variables)
|
||||
const configurations = useConfigurations(variables)
|
||||
const schema = useMemo(() => {
|
||||
return generateZodSchema(configurations)
|
||||
}, [configurations])
|
||||
|
||||
const form = useAppForm({
|
||||
defaultValues: initialData,
|
||||
validators: {
|
||||
onSubmit: ({ value }) => {
|
||||
const result = schema.safeParse(value)
|
||||
if (!result.success) {
|
||||
const issues = result.error.issues
|
||||
const firstIssue = issues[0]
|
||||
const errorMessage = `"${firstIssue!.path.join('.')}" ${firstIssue!.message}`
|
||||
toast.error(errorMessage)
|
||||
return errorMessage
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
},
|
||||
onSubmit: ({ value }) => {
|
||||
onSubmit(value)
|
||||
},
|
||||
})
|
||||
|
||||
const [fold, { toggle: foldToggle, setTrue: foldHide, setFalse: foldShow }] = useBoolean(false)
|
||||
|
||||
useEffect(() => {
|
||||
// When the step change
|
||||
if (step !== CrawlStep.init) foldHide()
|
||||
else foldShow()
|
||||
}, [step])
|
||||
|
||||
const isRunning = step === CrawlStep.running
|
||||
|
||||
return (
|
||||
<form
|
||||
className="w-full"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
form.handleSubmit()
|
||||
}}
|
||||
>
|
||||
<div className="flex items-center gap-x-1 px-4 py-2">
|
||||
<div
|
||||
className="flex grow cursor-pointer items-center gap-x-0.5 select-none"
|
||||
onClick={foldToggle}
|
||||
>
|
||||
<span className="system-sm-semibold-uppercase text-text-secondary">
|
||||
{t(($) => $[`${I18N_PREFIX}.options`], { ns: 'datasetCreation' })}
|
||||
</span>
|
||||
<span
|
||||
aria-hidden
|
||||
className={cn(
|
||||
'i-custom-vender-solid-general-arrow-down-round-fill h-4 w-4',
|
||||
cn('size-4 shrink-0 text-text-quaternary', fold && '-rotate-90'),
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
<Button
|
||||
variant="primary"
|
||||
onClick={form.handleSubmit}
|
||||
disabled={runDisabled}
|
||||
loading={isRunning}
|
||||
aria-labelledby={runButtonLabelId}
|
||||
className="shrink-0"
|
||||
>
|
||||
<RiPlayLargeLine aria-hidden className="size-4" />
|
||||
<span id={runButtonLabelId}>
|
||||
{!isRunning
|
||||
? t(($) => $[`${I18N_PREFIX}.run`], { ns: 'datasetCreation' })
|
||||
: t(($) => $[`${I18N_PREFIX}.running`], { ns: 'datasetCreation' })}
|
||||
</span>
|
||||
</Button>
|
||||
</div>
|
||||
{!fold && (
|
||||
<div className="flex flex-col gap-3 border-t border-divider-subtle px-4 py-3">
|
||||
{configurations.map((config, index) => {
|
||||
const FieldComponent = BaseField({
|
||||
initialData,
|
||||
config,
|
||||
})
|
||||
return <FieldComponent key={index} form={form} />
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</form>
|
||||
)
|
||||
}
|
||||
|
||||
export default Options
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
'use client'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
import type {
|
||||
DataSourceNodeCompletedResponse,
|
||||
DataSourceNodeErrorResponse,
|
||||
DataSourceNodeProcessingResponse,
|
||||
} from '@/types/pipeline'
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { useQueryState } from 'nuqs'
|
||||
import * as React from 'react'
|
||||
import { useCallback, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useShallow } from 'zustand/react/shallow'
|
||||
import {
|
||||
settingsQueryParamName,
|
||||
settingsQueryParser,
|
||||
} from '@/app/components/header/account-setting/query-params'
|
||||
import { selectPipelineStepParameters } from '@/app/components/rag-pipeline/utils/query-select'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { useDocLink } from '@/context/i18n'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import { consoleQuery } from '@/service/console'
|
||||
import Header from '../base/header'
|
||||
import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
|
||||
import CrawledResult from './base/crawled-result'
|
||||
import Crawling from './base/crawling'
|
||||
import ErrorMessage from './base/error-message'
|
||||
import Options from './base/options'
|
||||
|
||||
const I18N_PREFIX = 'stepOne.website'
|
||||
|
||||
type WebsiteCrawlProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
onCredentialChange: (credentialId: string) => void
|
||||
isInPipeline?: boolean
|
||||
supportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const WebsiteCrawl = ({
|
||||
nodeId,
|
||||
nodeData,
|
||||
isInPipeline = false,
|
||||
supportBatchUpload = true,
|
||||
onCredentialChange,
|
||||
}: WebsiteCrawlProps) => {
|
||||
const { t } = useTranslation(['datasetCreation'])
|
||||
const docLink = useDocLink()
|
||||
const [totalNum, setTotalNum] = useState(0)
|
||||
const [crawledNum, setCrawledNum] = useState(0)
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const pipelineId = useDatasetDetailContextWithSelector((s) => s.dataset?.pipeline_id)
|
||||
const [, setSettingsDestination] = useQueryState(settingsQueryParamName, settingsQueryParser)
|
||||
const { crawlResult, step, checkedCrawlResult, previewIndex, currentCredentialId } =
|
||||
useDataSourceStoreWithSelector(
|
||||
useShallow((state) => ({
|
||||
crawlResult: state.crawlResult,
|
||||
step: state.step,
|
||||
checkedCrawlResult: state.websitePages,
|
||||
previewIndex: state.previewIndex,
|
||||
currentCredentialId: state.currentCredentialId,
|
||||
})),
|
||||
)
|
||||
|
||||
const { data: dataSourceAuth } = useQuery(
|
||||
consoleQuery.auth.plugin.datasource.byProviderId.get.queryOptions({
|
||||
input: { params: { provider_id: `${nodeData.plugin_id}/${nodeData.provider_name}` } },
|
||||
}),
|
||||
)
|
||||
|
||||
const dataSourceStore = useDataSourceStore()
|
||||
|
||||
const preProcessingParamsOptions = {
|
||||
input: { params: { pipeline_id: pipelineId! }, query: { node_id: nodeId } },
|
||||
enabled: !!pipelineId && !!nodeId,
|
||||
select: selectPipelineStepParameters,
|
||||
}
|
||||
const workflows = consoleQuery.rag.pipelines.byPipelineId.workflows
|
||||
const { data: paramsConfig, isFetching: isFetchingParams } = useQuery(
|
||||
isInPipeline
|
||||
? workflows.draft.preProcessing.parameters.get.queryOptions(preProcessingParamsOptions)
|
||||
: workflows.published.preProcessing.parameters.get.queryOptions(preProcessingParamsOptions),
|
||||
)
|
||||
|
||||
const isInit = step === CrawlStep.init
|
||||
const isCrawlFinished = step === CrawlStep.finished
|
||||
const isRunning = step === CrawlStep.running
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const handleCheckedCrawlResultChange = useCallback(
|
||||
(checkedCrawlResult: CrawlResultItem[]) => {
|
||||
const { setWebsitePages } = dataSourceStore.getState()
|
||||
setWebsitePages(checkedCrawlResult)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const handlePreview = useCallback(
|
||||
(website: CrawlResultItem, index: number) => {
|
||||
const { setCurrentWebsite, setPreviewIndex } = dataSourceStore.getState()
|
||||
setCurrentWebsite(website)
|
||||
setPreviewIndex(index)
|
||||
},
|
||||
[dataSourceStore],
|
||||
)
|
||||
|
||||
const handleRun = useCallback(
|
||||
async (value: Record<string, any>) => {
|
||||
const { setStep, setCrawlResult, currentCredentialId } = dataSourceStore.getState()
|
||||
|
||||
setStep(CrawlStep.running)
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: value,
|
||||
datasource_type: DatasourceType.websiteCrawl,
|
||||
credential_id: currentCredentialId,
|
||||
response_mode: 'streaming',
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => {
|
||||
setTotalNum(data.total ?? 0)
|
||||
setCrawledNum(data.completed ?? 0)
|
||||
},
|
||||
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => {
|
||||
const { data: crawlData, time_consuming } = data
|
||||
const crawlResultData = {
|
||||
data: crawlData as CrawlResultItem[],
|
||||
time_consuming: time_consuming ?? 0,
|
||||
}
|
||||
setCrawlResult(crawlResultData)
|
||||
handleCheckedCrawlResultChange(supportBatchUpload ? crawlData : crawlData.slice(0, 1)) // default select the crawl result
|
||||
setCrawlErrorMessage('')
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
|
||||
setCrawlErrorMessage(
|
||||
error.error || t(($) => $[`${I18N_PREFIX}.unknownError`], { ns: 'datasetCreation' }),
|
||||
)
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
},
|
||||
)
|
||||
},
|
||||
[dataSourceStore, datasourceNodeRunURL, handleCheckedCrawlResultChange, supportBatchUpload, t],
|
||||
)
|
||||
|
||||
const handleSubmit = useCallback(
|
||||
(value: Record<string, any>) => {
|
||||
handleRun(value)
|
||||
},
|
||||
[handleRun],
|
||||
)
|
||||
|
||||
const handleSetting = useCallback(() => {
|
||||
setSettingsDestination('data-source')
|
||||
}, [setSettingsDestination])
|
||||
|
||||
const handleCredentialChange = useCallback(
|
||||
(credentialId: string) => {
|
||||
setCrawledNum(0)
|
||||
setTotalNum(0)
|
||||
setCrawlErrorMessage('')
|
||||
onCredentialChange(credentialId)
|
||||
},
|
||||
[onCredentialChange],
|
||||
)
|
||||
|
||||
return (
|
||||
<div className="flex flex-col">
|
||||
<Header
|
||||
docTitle="Docs"
|
||||
docLink={docLink('/use-dify/knowledge/knowledge-pipeline/authorize-data-source')}
|
||||
onClickConfiguration={handleSetting}
|
||||
pluginName={nodeData.datasource_label}
|
||||
currentCredentialId={currentCredentialId}
|
||||
onCredentialChange={handleCredentialChange}
|
||||
credentials={dataSourceAuth?.result || []}
|
||||
/>
|
||||
<div className="mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle">
|
||||
<Options
|
||||
variables={paramsConfig?.variables || []}
|
||||
step={step}
|
||||
runDisabled={!currentCredentialId || isFetchingParams}
|
||||
onSubmit={handleSubmit}
|
||||
/>
|
||||
</div>
|
||||
{!isInit && (
|
||||
<div className="relative flex flex-col">
|
||||
{isRunning && <Crawling crawledNum={crawledNum} totalNum={totalNum} />}
|
||||
{showError && (
|
||||
<ErrorMessage
|
||||
className="mt-2"
|
||||
title={t(($) => $[`${I18N_PREFIX}.exceptionErrorTitle`], { ns: 'datasetCreation' })}
|
||||
errorMsg={crawlErrorMessage}
|
||||
/>
|
||||
)}
|
||||
{isCrawlFinished && !showError && (
|
||||
<CrawledResult
|
||||
className="mt-2"
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={handleCheckedCrawlResultChange}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
previewIndex={previewIndex}
|
||||
onPreview={handlePreview}
|
||||
showPreview={!isInPipeline}
|
||||
isMultipleChoice={supportBatchUpload} // only support single choice in test run
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(WebsiteCrawl)
|
||||
Reference in new issue
Block a user