chore: 纳入本地 Dify 1.19.0 源码及品牌与免登改动

演示目录只保留可重建的源码。运行密钥、数据卷、依赖和带口令的本地配置不进入仓库。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
wuzilongandCursor committed 2026-11-18 13:59:25 +08:00
1 parent 60158935f4
commit 6519682885
15831 files changed
+3063428

No files matched your search

@@ -0,0 +1,223 @@
import type { CustomRunFormProps, DataSourceNodeType } from '../types'
import type { DataSourceShape } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import { screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { useStoreApi } from 'reactflow'
import {
useDataSourceStore,
useDataSourceStoreWithSelector,
} from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import { createNode } from '@/app/components/workflow/__tests__/fixtures'
import { renderWorkflowFlowComponent } from '@/app/components/workflow/__tests__/workflow-test-env'
import { DatasourceType } from '@/models/pipeline'
import { BlockEnum, NodeRunningStatus } from '../../../types'
import BeforeRunForm from '../before-run-form'
const run = vi.hoisted(() => vi.fn())
vi.mock('@/service/console', async (importOriginal) => {
const actual = await importOriginal<typeof import('@/service/console')>()
const { withConsoleResponses } = await import('@/test/console/service-responses')
return {
...actual,
consoleQuery: withConsoleResponses(actual.consoleQuery, {
'rag.pipelines.byPipelineId.workflows.draft.datasource.variablesInspect.post': run,
}),
}
})
vi.mock('@/service/workflow', async (importOriginal) => ({
...(await importOriginal<typeof import('@/service/workflow')>()),
fetchNodeInspectVars: async () => [],
}))
const document: DataSourceShape['onlineDocuments'][number] = {
workspace_id: 'workspace',
page_id: 'page',
page_name: 'Guide',
page_icon: null,
parent_id: '',
type: 'page',
is_bound: false,
}
const website: DataSourceShape['websitePages'][number] = {
source_url: 'https://example.com',
title: 'Guide',
markdown: '# Guide',
description: '',
}
const localFiles: DataSourceShape['localFileList'] = [
{
fileID: 'upload',
progress: 100,
file: Object.assign(new File(['pdf'], 'guide.pdf', { type: 'application/pdf' }), {
id: 'file-1',
extension: 'pdf',
mime_type: 'application/pdf',
}),
},
]
// Uploading/browsing belongs to the independently tested datasource children.
// These controls exercise their public selection and credential callbacks against the real store.
function Selection({
type,
onCredentialChange,
allowedExtensions,
}: {
type: DatasourceType
onCredentialChange?: (id: string) => void
allowedExtensions?: string[]
}) {
const store = useDataSourceStore()
const credential = useDataSourceStoreWithSelector((state) => state.currentCredentialId)
const selected = useDataSourceStoreWithSelector(
(state) =>
state.localFileList.length +
state.onlineDocuments.length +
state.websitePages.length +
state.selectedFileIds.length,
)
const select = () => {
const state = store.getState()
if (type === DatasourceType.localFile) state.setLocalFileList(localFiles)
if (type === DatasourceType.onlineDocument) {
state.setOnlineDocuments([document])
state.setSelectedPagesId(new Set(['page']))
state.setCurrentDocument(document)
}
if (type === DatasourceType.websiteCrawl) {
state.setWebsitePages([website])
state.setCurrentWebsite(website)
state.setPreviewIndex(0)
}
if (type === DatasourceType.onlineDrive) {
state.setOnlineDriveFileList([{ id: 'drive-file', name: 'Guide', type: 'file' }])
state.setBucket('bucket')
state.setSelectedFileIds(['drive-file'])
}
}
return (
<>
<span>{allowedExtensions?.join(',')}</span>
<button type="button" onClick={select}>
Select Guide
</button>
<output aria-label="Selected documents">{selected}</output>
<output aria-label="Credential">{credential}</output>
{onCredentialChange && (
<button type="button" onClick={() => onCredentialChange('new-credential')}>
Switch credential
</button>
)}
</>
)
}
vi.mock('@/app/components/datasets/documents/create-from-pipeline/data-source/local-file', () => ({
default: (props: { allowedExtensions: string[] }) => (
<Selection {...props} type={DatasourceType.localFile} />
),
}))
vi.mock(
'@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents',
() => ({
default: (props: { onCredentialChange: (id: string) => void }) => (
<Selection {...props} type={DatasourceType.onlineDocument} />
),
}),
)
vi.mock(
'@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl',
() => ({
default: (props: { onCredentialChange: (id: string) => void }) => (
<Selection {...props} type={DatasourceType.websiteCrawl} />
),
}),
)
vi.mock(
'@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive',
() => ({
default: (props: { onCredentialChange: (id: string) => void }) => (
<Selection {...props} type={DatasourceType.onlineDrive} />
),
}),
)
const createData = (overrides: Partial<DataSourceNodeType> = {}): DataSourceNodeType => ({
title: 'Datasource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: 'plugin-id',
provider_type: DatasourceType.localFile,
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'Local File',
datasource_parameters: {},
datasource_configurations: {},
fileExtensions: ['pdf', 'md'],
...overrides,
})
const createProps = (overrides: Partial<CustomRunFormProps> = {}): CustomRunFormProps => ({
nodeId: 'data-source-node',
resource: { kind: 'knowledge-pipeline', pipelineId: 'flow-id' },
payload: createData(),
setRunResult: vi.fn(),
setIsRunAfterSingleRun: vi.fn(),
isPaused: false,
isRunAfterSingleRun: false,
onSuccess: vi.fn(),
onCancel: vi.fn(),
appendNodeInspectVars: vi.fn(),
...overrides,
})
function setup(props = createProps()) {
let flow: ReturnType<typeof useStoreApi>
function Form() {
flow = useStoreApi()
return <BeforeRunForm {...props} />
}
renderWorkflowFlowComponent(<Form />, {
resource: props.resource,
nodes: [createNode({ id: props.nodeId, data: props.payload })],
hooksStoreProps: { doSyncWorkflowDraft: async (_skip, callbacks) => callbacks?.onSuccess?.() },
})
return { props, nodeData: () => flow.getState().getNodes()[0]!.data }
}
beforeEach(() => run.mockResolvedValue({ id: 'run-1', status: NodeRunningStatus.Succeeded }))
it('runs the selected local file and exposes cancel through the form', async () => {
const user = userEvent.setup()
const { props, nodeData } = setup()
expect(screen.getByText(/Datasource/)).toBeInTheDocument()
expect(screen.getByText('pdf,md')).toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'Select Guide' }))
await user.click(screen.getByRole('button', { name: 'common.operation.cancel' }))
await user.click(screen.getByRole('button', { name: 'workflow.singleRun.startRun' }))
await waitFor(() => expect(nodeData()._singleRunningStatus).toBe(NodeRunningStatus.Succeeded))
expect(run).toHaveBeenCalledWith({
params: { pipeline_id: 'flow-id' },
body: expect.objectContaining({
start_node_id: props.nodeId,
datasource_type: DatasourceType.localFile,
datasource_info: expect.objectContaining({ related_id: 'file-1', name: 'guide.pdf' }),
}),
})
expect(props.onCancel).toHaveBeenCalledOnce()
})
it.each([DatasourceType.onlineDocument, DatasourceType.websiteCrawl, DatasourceType.onlineDrive])(
'clears the previous %s selection when credentials change',
async (type) => {
const user = userEvent.setup()
setup(createProps({ payload: createData({ provider_type: type }) }))
await user.click(screen.getByRole('button', { name: 'Select Guide' }))
expect(screen.getByRole('status', { name: 'Selected documents' })).toHaveTextContent('1')
expect(screen.getByRole('button', { name: 'workflow.singleRun.startRun' })).toBeEnabled()
await user.click(screen.getByRole('button', { name: 'Switch credential' }))
expect(screen.getByRole('status', { name: 'Credential' })).toHaveTextContent('new-credential')
expect(screen.getByRole('status', { name: 'Selected documents' })).toHaveTextContent('0')
expect(screen.getByRole('button', { name: 'workflow.singleRun.startRun' })).toBeDisabled()
},
)
@@ -0,0 +1,139 @@
import type { DataSourceNodeType } from '../types'
import { useQuery } from '@tanstack/react-query'
import { act, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { renderWorkflowComponent } from '@/app/components/workflow/__tests__/workflow-test-env'
import { BlockEnum } from '@/app/components/workflow/types'
import { consoleQuery } from '@/service/console'
import { createConsoleQueryWrapper } from '@/test/console/query-data'
import { createPluginEntity } from '@/test/fixtures/plugin'
import Node from '../node'
const requests = vi.hoisted(() => ({
manifest: vi.fn(),
install: vi.fn(),
catalog: vi.fn(),
}))
vi.mock('@/service/console', async (importOriginal) => {
const actual = await importOriginal<typeof import('@/service/console')>()
const { withConsoleResponses, withConsoleClientResponses } =
await import('@/test/console/service-responses')
return {
...actual,
consoleQuery: withConsoleResponses(actual.consoleQuery, {
'workspaces.current.plugin.list.installations.ids.post': requests.manifest,
'rag.pipelines.datasourcePlugins.get': requests.catalog,
}),
consoleClient: withConsoleClientResponses(actual.consoleClient, {
'workspaces.current.plugin.install.marketplace.post': requests.install,
}),
}
})
const provider = createDatasourceProvider()
const data: DataSourceNodeType = {
title: 'Datasource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: provider.plugin_id,
provider_type: 'local_file',
provider_name: provider.provider,
datasource_name: 'local-file',
datasource_label: 'Local File',
datasource_parameters: {},
datasource_configurations: {},
plugin_unique_identifier: provider.plugin_unique_identifier,
}
// Subscribe to the pipeline catalog so invalidation must deliver fresh data.
function DatasourceCatalog() {
const { data: providers } = useQuery(
consoleQuery.rag.pipelines.datasourcePlugins.get.queryOptions(),
)
return (
<output aria-label="Datasource catalog">
{providers?.map((item) => item.provider).join(', ')}
</output>
)
}
beforeEach(() => {
requests.manifest.mockReset().mockResolvedValue({ plugins: [] })
requests.install.mockReset().mockResolvedValue({ all_installed: true, task_id: '' })
requests.catalog.mockReset().mockResolvedValue([])
})
it('installs the missing datasource package, hides installation and refreshes the subscribed catalog', async () => {
const user = userEvent.setup()
const { queryClient } = createConsoleQueryWrapper({ workspacePermissionKeys: ['plugin.install'] })
queryClient.setQueryData(consoleQuery.rag.pipelines.datasourcePlugins.get.queryKey(), [])
requests.manifest.mockResolvedValueOnce({ plugins: [] }).mockResolvedValue({
plugins: [
createPluginEntity({
plugin_id: provider.plugin_id,
plugin_unique_identifier: provider.plugin_unique_identifier,
}),
],
})
requests.install.mockImplementation(async () => {
requests.catalog.mockResolvedValue([provider])
return { all_installed: true, task_id: '' }
})
renderWorkflowComponent(
<>
<Node id="data-source-node" data={data} />
<DatasourceCatalog />
</>,
{
queryClient,
initialStoreState: { dataSourceList: [] },
},
)
// Settle the initial empty catalog fetch before installation can change the server response.
await act(async () => {
await queryClient.query(consoleQuery.rag.pipelines.datasourcePlugins.get.queryOptions())
})
expect(screen.getByLabelText('Datasource catalog')).toBeEmptyDOMElement()
await user.click(
await screen.findByRole('button', {
name: 'workflowAgent.nodes.agent.pluginInstaller.install',
}),
)
expect(requests.install).toHaveBeenCalledExactlyOnceWith({
body: { plugin_unique_identifiers: [provider.plugin_unique_identifier] },
})
await waitFor(() => expect(screen.getByLabelText('Datasource catalog')).toHaveTextContent('file'))
await waitFor(() =>
expect(
screen.queryByRole('button', { name: /pluginInstaller.install/ }),
).not.toBeInTheDocument(),
)
})
it('does not offer installation without workspace install permission', () => {
const { queryClient } = createConsoleQueryWrapper({ workspacePermissionKeys: [] })
const pluginIds = Array.from(
new Set(
[data.plugin_unique_identifier, data.plugin_id, data.provider_name].filter(
(item): item is string => Boolean(item),
),
),
)
// Resolve the manifest before render so query loading cannot hide a permission regression.
queryClient.setQueryData(
consoleQuery.workspaces.current.plugin.list.installations.ids.post.queryKey({
input: { body: { plugin_ids: pluginIds } },
}),
{ plugins: [] },
)
renderWorkflowComponent(<Node id="data-source-node" data={data} />, {
queryClient,
initialStoreState: { dataSourceList: [] },
})
expect(screen.queryByRole('button', { name: /pluginInstaller.install/ })).not.toBeInTheDocument()
expect(requests.install).not.toHaveBeenCalled()
})
@@ -0,0 +1,157 @@
import type { DataSourceNodeType } from '../types'
import { screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { useNodes } from 'reactflow'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { renderWorkflowFlowComponent } from '@/app/components/workflow/__tests__/workflow-test-env'
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
import { BlockEnum } from '@/app/components/workflow/types'
import Panel from '../panel'
const panelProps = {
getInputVars: () => [],
toVarInputs: () => [],
runInputData: {},
runInputDataRef: { current: {} },
setRunInputData: () => {},
runResult: null,
}
const createData = (overrides: Partial<DataSourceNodeType> = {}): DataSourceNodeType => ({
title: 'Datasource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: 'langgenius/file',
provider_type: 'online_document',
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'File',
datasource_parameters: {},
datasource_configurations: {},
...overrides,
})
const { get } = vi.hoisted(() => ({
get: vi.fn(async (_url: string) => []),
}))
vi.mock('@/service/base', async (importOriginal) => ({
...(await importOriginal<typeof import('@/service/base')>()),
get,
// Console contract requests; only the schema type definitions are expected.
request: async (_url: string, _init: RequestInit, { request }: { request: Request }) => {
if (new URL(request.url).pathname.endsWith('/spec/schema-definitions')) return Response.json([])
throw new Error(`Unexpected request: ${request.url}`)
},
}))
const createProvider = () => {
const provider = createDatasourceProvider()
provider.declaration.datasources![0]!.parameters = [
{
name: 'count',
type: 'number',
required: true,
default: 0,
min: 0,
max: 10,
label: { en_US: 'Count', zh_Hans: null },
description: { en_US: 'How many documents' },
},
{
name: 'enabled',
type: 'boolean',
required: false,
default: false,
label: { en_US: 'Enabled' },
description: { en_US: 'Include archived documents' },
},
]
return provider
}
function EditablePanel() {
const node = useNodes<DataSourceNodeType>().find((node) => node.id === 'datasource')
if (!node) return null
return (
<>
<Panel id={node.id} data={node.data} panelProps={panelProps} />
<output aria-label="Datasource parameters">
{JSON.stringify(node.data.datasource_parameters)}
</output>
</>
)
}
it('renders real datasource fields and descriptions without tool dynamic requests, preserving numeric and boolean values', async () => {
const user = userEvent.setup()
const data = createData({
datasource_parameters: {
count: { type: VarKindType.constant, value: 0 },
enabled: { type: VarKindType.constant, value: false },
},
})
renderWorkflowFlowComponent(<EditablePanel />, {
nodes: [{ id: 'datasource', position: { x: 0, y: 0 }, data }],
initialStoreState: { dataSourceList: [createProvider()] },
})
expect(screen.getByText('How many documents')).toBeInTheDocument()
const count = screen.getByRole('textbox', { name: 'Count' })
expect(count).toHaveValue('0')
await user.click(count)
await user.keyboard('{ArrowDown}')
expect(count).toHaveValue('0')
await user.clear(count)
await user.type(count, '2')
expect(screen.getByRole('status', { name: 'Datasource parameters' })).toHaveTextContent(
'"count":{"type":"constant","value":2}',
)
await user.click(screen.getByRole('button', { name: 'True' }))
expect(screen.getByRole('status', { name: 'Datasource parameters' })).toHaveTextContent(
JSON.stringify({
count: { type: VarKindType.constant, value: 2 },
enabled: { type: VarKindType.constant, value: true },
}),
)
expect(get.mock.calls.map((call) => call[0])).not.toContain(
'/workspaces/current/plugin/parameters/dynamic-options',
)
})
it('renders nested and unrestricted outputs while keeping unauthorized inputs hidden', async () => {
const user = userEvent.setup()
const provider = createProvider()
provider.is_authorized = false
provider.declaration.datasources![0]!.output_schema = {
properties: {
metadata: {
type: 'object',
properties: { title: { type: 'string' }, optional: true, nullable: null },
},
names: { type: 'array', items: { type: 'string' } },
unknown: false,
},
}
const data = createData()
renderWorkflowFlowComponent(<Panel id="datasource" data={data} panelProps={panelProps} />, {
nodes: [{ id: 'datasource', position: { x: 0, y: 0 }, data }],
initialStoreState: { dataSourceList: [provider] },
})
await user.click(screen.getByRole('button', { name: 'workflow.nodes.common.outputVars' }))
await waitFor(() => expect(screen.getByText('metadata')).toBeInTheDocument())
expect(screen.getByText('title')).toBeInTheDocument()
expect(screen.getByText('optional')).toBeInTheDocument()
expect(screen.getByText('nullable')).toBeInTheDocument()
expect(screen.getByText('array[string]')).toBeInTheDocument()
expect(screen.queryByRole('textbox', { name: 'Count' })).not.toBeInTheDocument()
})
it('keeps local-file controls available without plugin version metadata', async () => {
const user = userEvent.setup()
const data = createData({ provider_type: 'local_file', fileExtensions: ['pdf', 'csv'] })
renderWorkflowFlowComponent(<Panel id="datasource" data={data} panelProps={panelProps} />, {
nodes: [{ id: 'datasource', position: { x: 0, y: 0 }, data }],
initialStoreState: { dataSourceList: [createDatasourceProvider()] },
})
expect(screen.getByText('pdf')).toBeInTheDocument()
expect(screen.getByText('csv')).toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'workflow.nodes.common.outputVars' }))
expect(screen.getByText('datasource_type')).toBeInTheDocument()
expect(screen.getByText('transfer_method')).toBeInTheDocument()
})
@@ -0,0 +1,115 @@
'use client'
import type { FC } from 'react'
import type { CustomRunFormProps } from './types'
import { Button } from '@langgenius/dify-ui/button'
import * as React from 'react'
import { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
import OnlineDrive from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive'
import { useDataSourceStore } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import DataSourceProvider from '@/app/components/datasets/documents/create-from-pipeline/data-source/store/provider'
import WebsiteCrawl from '@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl'
import {
useOnlineDocument,
useOnlineDrive,
useWebsiteCrawl,
} from '@/app/components/rag-pipeline/components/panel/test-run/preparation/hooks'
import { DatasourceType } from '@/models/pipeline'
import PanelWrap from '../_base/components/before-run-form/panel-wrap'
import useBeforeRunForm from './hooks/use-before-run-form'
const BeforeRunForm: FC<CustomRunFormProps> = (props) => {
const { nodeId, payload, onCancel } = props
const { t } = useTranslation(['common', 'workflow'])
const dataSourceStore = useDataSourceStore()
const {
isPending,
handleRunWithSyncDraft,
datasourceType,
datasourceNodeData,
startRunBtnDisabled,
} = useBeforeRunForm(props)
const { clearOnlineDocumentData } = useOnlineDocument()
const { clearWebsiteCrawlData } = useWebsiteCrawl()
const { clearOnlineDriveData } = useOnlineDrive()
const clearDataSourceData = useCallback(() => {
if (datasourceType === DatasourceType.onlineDocument) clearOnlineDocumentData()
else if (datasourceType === DatasourceType.websiteCrawl) clearWebsiteCrawlData()
else if (datasourceType === DatasourceType.onlineDrive) clearOnlineDriveData()
}, [clearOnlineDocumentData, clearOnlineDriveData, clearWebsiteCrawlData, datasourceType])
const handleCredentialChange = useCallback(
(credentialId: string) => {
const { setCurrentCredentialId } = dataSourceStore.getState()
clearDataSourceData()
setCurrentCredentialId(credentialId)
},
[clearDataSourceData, dataSourceStore],
)
return (
<PanelWrap nodeName={payload.title} onHide={onCancel}>
<div className="flex flex-col gap-y-5 px-4 pt-4">
{datasourceType === DatasourceType.localFile && (
<LocalFile
allowedExtensions={datasourceNodeData.fileExtensions || []}
supportBatchUpload={false}
/>
)}
{datasourceType === DatasourceType.onlineDocument && (
<OnlineDocuments
nodeId={nodeId}
nodeData={datasourceNodeData}
isInPipeline
onCredentialChange={handleCredentialChange}
supportBatchUpload={false}
/>
)}
{datasourceType === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={nodeId}
nodeData={datasourceNodeData}
isInPipeline
onCredentialChange={handleCredentialChange}
supportBatchUpload={false}
/>
)}
{datasourceType === DatasourceType.onlineDrive && (
<OnlineDrive
nodeId={nodeId}
nodeData={datasourceNodeData}
isInPipeline
onCredentialChange={handleCredentialChange}
supportBatchUpload={false}
/>
)}
<div className="flex justify-end gap-x-2">
<Button onClick={onCancel}>{t(($) => $['operation.cancel'], { ns: 'common' })}</Button>
<Button
onClick={handleRunWithSyncDraft}
variant="primary"
loading={isPending}
disabled={startRunBtnDisabled}
>
{t(($) => $['singleRun.startRun'], { ns: 'workflow' })}
</Button>
</div>
</div>
</PanelWrap>
)
}
const BeforeRunFormWrapper = (props: CustomRunFormProps) => {
return (
<DataSourceProvider>
<BeforeRunForm {...props} />
</DataSourceProvider>
)
}
export default React.memo(BeforeRunFormWrapper)
@@ -0,0 +1,59 @@
import { VarType } from '@/app/components/workflow/types'
export const COMMON_OUTPUT = [
{
name: 'datasource_type',
type: VarType.string,
description: 'local_file, online_document, website_crawl',
},
]
export const LOCAL_FILE_OUTPUT = [
{
name: 'file',
type: VarType.file,
description: 'file',
subItems: [
{
name: 'name',
type: VarType.string,
description: 'file name',
},
{
name: 'size',
type: VarType.number,
description: 'file size',
},
{
name: 'type',
type: VarType.string,
description: 'file type',
},
{
name: 'extension',
type: VarType.string,
description: 'file extension',
},
{
name: 'mime_type',
type: VarType.string,
description: 'file mime type',
},
{
name: 'transfer_method',
type: VarType.string,
description: 'file transfer method',
},
{
name: 'url',
type: VarType.string,
description: 'file url',
},
{
name: 'related_id',
type: VarType.string,
description: 'file related id',
},
],
},
]
@@ -0,0 +1,127 @@
import type { SchemaDefinitionItemResponse } from '@dify/contracts/api/console/spec/types.gen'
import type { TFunction } from 'i18next'
import type { NodeDefault, Var } from '../../types'
import type { getDataSourceCheckParams } from '../../utils/data-source'
import type { DataSourceNodeType } from './types'
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
import { BlockEnum } from '@/app/components/workflow/types'
import { genNodeMetaData } from '@/app/components/workflow/utils'
import { matchDataSource } from '../../utils/plugin-install-check'
import { resolveVarType } from '../tool/output-schema-utils'
import { COMMON_OUTPUT, LOCAL_FILE_OUTPUT } from './constants'
import { DataSourceClassification } from './types'
const i18nPrefix = 'errorMsg'
const metaData = genNodeMetaData({
sort: -1,
type: BlockEnum.DataSource,
isStart: true,
isRequired: true,
})
const getOutputProperties = (
schema: unknown,
schemaTypeDefinitions?: SchemaDefinitionItemResponse[],
): Var[] => {
if (!schema || typeof schema !== 'object' || !('properties' in schema)) return []
const properties = schema.properties
if (!properties || typeof properties !== 'object' || Array.isArray(properties)) return []
return Object.entries(properties).map(([variable, value]: [string, unknown]) => {
const schemaValue =
value && typeof value === 'object' && !Array.isArray(value) ? value : undefined
const { type, schemaType } = resolveVarType(schemaValue, schemaTypeDefinitions)
return {
variable,
type,
schemaType,
des:
value &&
typeof value === 'object' &&
'description' in value &&
typeof value.description === 'string'
? value.description
: undefined,
children: type === 'object' ? getOutputProperties(value, schemaTypeDefinitions) : undefined,
}
})
}
const nodeDefault: NodeDefault<DataSourceNodeType> = {
metaData,
defaultValue: {
datasource_parameters: {},
datasource_configurations: {},
},
checkValid(
payload,
t: TFunction<['workflow']>,
moreDataForCheckValid: ReturnType<typeof getDataSourceCheckParams>,
) {
const { dataSourceInputsSchema, notAuthed } = moreDataForCheckValid
let errorMessage = ''
if (notAuthed) errorMessage = t(($) => $[`${i18nPrefix}.authRequired`], { ns: 'workflow' })
if (!errorMessage) {
dataSourceInputsSchema
.filter((field) => {
return field.required
})
.forEach((field) => {
const targetVar = payload.datasource_parameters[field.variable]
if (!targetVar) {
errorMessage = t(($) => $[`${i18nPrefix}.fieldRequired`], {
ns: 'workflow',
field: field.label,
})
return
}
const { type: variable_type, value } = targetVar
if (variable_type === VarKindType.variable) {
if (!errorMessage && (!value || value.length === 0))
errorMessage = t(($) => $[`${i18nPrefix}.fieldRequired`], {
ns: 'workflow',
field: field.label,
})
} else {
if (!errorMessage && (value === undefined || value === null || value === ''))
errorMessage = t(($) => $[`${i18nPrefix}.fieldRequired`], {
ns: 'workflow',
field: field.label,
})
}
})
}
return {
isValid: !errorMessage,
errorMessage,
}
},
getOutputVars(
payload,
allPluginInfoList,
ragVars = [],
{ schemaTypeDefinitions } = { schemaTypeDefinitions: [] },
) {
const { datasource_name, provider_type } = payload
const isLocalFile = provider_type === DataSourceClassification.localFile
const provider = matchDataSource(allPluginInfoList.dataSourceList ?? [], payload)
const datasource = provider?.declaration.datasources?.find(
(item) => item.identity.name === datasource_name,
)
const dynamicOutputSchema = getOutputProperties(
datasource?.output_schema,
schemaTypeDefinitions,
)
return [
...COMMON_OUTPUT.map((item) => ({ variable: item.name, type: item.type })),
...(isLocalFile
? LOCAL_FILE_OUTPUT.map((item) => ({ variable: item.name, type: item.type }))
: []),
...ragVars,
...dynamicOutputSchema,
]
},
}
export default nodeDefault
@@ -0,0 +1,311 @@
import type { WorkflowRunNodeExecutionResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { ReactNode } from 'react'
import type { CustomRunFormProps, DataSourceNodeType } from '../../types'
import type { DataSourceShape } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import type { Shape as HooksStore } from '@/app/components/workflow/hooks-store/store'
import { QueryClient } from '@tanstack/react-query'
import { act, waitFor } from '@testing-library/react'
import { useStoreApi } from 'reactflow'
import { createDataSourceStore } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import { DataSourceContext } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store/provider'
import { createNode } from '@/app/components/workflow/__tests__/fixtures'
import { renderWorkflowCanvasHook } from '@/app/components/workflow/__tests__/workflow-test-env'
import { BlockEnum, NodeRunningStatus } from '@/app/components/workflow/types'
import { DatasourceType } from '@/models/pipeline'
import { consoleQuery } from '@/service/console'
import { createDeferred } from '@/test/deferred'
import { TransferMethod } from '@/types/app'
import useBeforeRunForm from '../use-before-run-form'
const mocks = vi.hoisted(() => ({
run: vi.fn(),
inspect: vi.fn(),
sync: vi.fn<HooksStore['doSyncWorkflowDraft']>(),
}))
vi.mock('@/service/console', async (importOriginal) => {
const actual = await importOriginal<typeof import('@/service/console')>()
const { withConsoleResponses } = await import('@/test/console/service-responses')
return {
...actual,
consoleQuery: withConsoleResponses(actual.consoleQuery, {
'rag.pipelines.byPipelineId.workflows.draft.datasource.variablesInspect.post': mocks.run,
}),
}
})
vi.mock('@/service/workflow', async (importOriginal) => ({
...(await importOriginal<typeof import('@/service/workflow')>()),
fetchNodeInspectVars: mocks.inspect,
}))
const createData = (overrides: Partial<DataSourceNodeType> = {}): DataSourceNodeType => ({
title: 'Datasource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: 'plugin-id',
provider_type: DatasourceType.localFile,
provider_name: 'provider',
datasource_name: 'local-file',
datasource_label: 'Local File',
datasource_parameters: {},
datasource_configurations: {},
fileExtensions: ['pdf'],
...overrides,
})
const createProps = (overrides: Partial<CustomRunFormProps> = {}): CustomRunFormProps => ({
nodeId: 'data-source-node',
resource: { kind: 'knowledge-pipeline', pipelineId: 'flow-id' },
payload: createData(),
setRunResult: vi.fn(),
setIsRunAfterSingleRun: vi.fn(),
isPaused: false,
isRunAfterSingleRun: false,
onSuccess: vi.fn(),
onCancel: vi.fn(),
appendNodeInspectVars: vi.fn(),
...overrides,
})
const localFiles: DataSourceShape['localFileList'] = [
{
fileID: 'upload-1',
progress: 100,
file: Object.assign(new File(['pdf'], 'doc.pdf', { type: 'application/pdf' }), {
id: 'file-1',
extension: 'pdf',
mime_type: 'application/pdf',
}),
},
]
const document: DataSourceShape['onlineDocuments'][number] = {
workspace_id: 'workspace-1',
page_id: 'page-1',
page_name: 'Guide',
page_icon: null,
parent_id: '',
type: 'page',
is_bound: false,
}
const website: DataSourceShape['websitePages'][number] = {
source_url: 'https://example.com/guide',
title: 'Guide',
markdown: '# Guide',
description: 'A guide',
}
const selections = [
{
type: DatasourceType.localFile,
state: { localFileList: localFiles },
info: {
related_id: 'file-1',
name: 'doc.pdf',
type: 'application/pdf',
size: 3,
extension: 'pdf',
mime_type: 'application/pdf',
url: '',
transfer_method: TransferMethod.local_file,
},
},
{
type: DatasourceType.onlineDocument,
state: { onlineDocuments: [document] },
info: {
workspace_id: 'workspace-1',
page: {
page_id: 'page-1',
page_name: 'Guide',
page_icon: null,
parent_id: '',
type: 'page',
is_bound: false,
},
credential_id: 'credential-1',
},
},
{
type: DatasourceType.websiteCrawl,
state: { websitePages: [website] },
info: { ...website, credential_id: 'credential-1' },
},
{
type: DatasourceType.onlineDrive,
state: {
selectedFileIds: ['drive-1'],
onlineDriveFileList: [{ id: 'drive-1', name: 'Guide', type: 'file' as const }],
bucket: 'bucket-1',
},
info: {
bucket: 'bucket-1',
id: 'drive-1',
type: 'file',
credential_id: 'credential-1',
},
},
] satisfies {
type: DatasourceType
state: Partial<DataSourceShape>
info: Record<string, unknown>
}[]
function setup(props = createProps(), selection: Partial<DataSourceShape> = {}) {
const store = createDataSourceStore()
store.setState({ currentCredentialId: 'credential-1', ...selection })
const client = new QueryClient({
defaultOptions: { queries: { retry: false }, mutations: { retry: false } },
})
const lastRunKey =
consoleQuery.rag.pipelines.byPipelineId.workflows.draft.nodes.byNodeId.lastRun.get.queryKey({
input: { params: { pipeline_id: 'flow-id', node_id: props.nodeId } },
})
client.setQueryData(lastRunKey, { id: 'last-run-1', status: 'succeeded' })
const wrapper = ({ children }: { children: ReactNode }) => (
<DataSourceContext value={store}>{children}</DataSourceContext>
)
let flow: ReturnType<typeof useStoreApi>
const rendered = renderWorkflowCanvasHook(
(props: CustomRunFormProps) => {
flow = useStoreApi()
return useBeforeRunForm(props)
},
{
initialProps: props,
wrapper,
queryClient: client,
resource: props.resource,
nodes: [createNode({ id: props.nodeId, data: props.payload })],
hooksStoreProps: { doSyncWorkflowDraft: mocks.sync },
},
)
return {
...rendered,
nodeData: () => flow.getState().getNodes()[0]!.data,
store,
props,
client,
lastRunKey,
}
}
beforeEach(() => {
vi.resetAllMocks()
mocks.run.mockResolvedValue({ id: 'run-1', status: NodeRunningStatus.Succeeded })
mocks.inspect.mockResolvedValue([])
mocks.sync.mockImplementation(async (_skipRefresh, callbacks) => callbacks?.onSuccess?.())
})
describe('data source single run', () => {
it.each(selections)(
'reacts to selection changes and sends the $type payload',
async ({ type, state, info }) => {
const { result, store, props, nodeData } = setup(
createProps({ payload: createData({ provider_type: type }) }),
)
expect(result.current.startRunBtnDisabled).toBe(true)
act(() => store.setState(state))
expect(result.current.startRunBtnDisabled).toBe(false)
act(() => result.current.handleRunWithSyncDraft())
await waitFor(() => expect(props.onSuccess).toHaveBeenCalledOnce())
expect(mocks.run).toHaveBeenCalledExactlyOnceWith({
params: { pipeline_id: 'flow-id' },
body: {
start_node_id: props.nodeId,
start_node_title: 'Datasource',
datasource_type: type,
datasource_info: info,
},
})
expect(mocks.inspect).toHaveBeenCalledWith(props.resource, props.nodeId)
expect(props.appendNodeInspectVars).toHaveBeenCalledWith(
props.nodeId,
[],
[expect.objectContaining({ id: props.nodeId })],
)
expect(props.setIsRunAfterSingleRun).toHaveBeenCalledWith(true)
expect(nodeData()).toMatchObject({
_singleRunningStatus: NodeRunningStatus.Succeeded,
_isSingleRun: false,
})
},
)
it('keeps local uploads disabled until an uploaded file id exists', () => {
const { result, store } = setup(createProps(), {
localFileList: [{ fileID: 'pending', progress: 20, file: new File(['pdf'], 'doc.pdf') }],
})
expect(result.current.startRunBtnDisabled).toBe(true)
act(() => store.getState().setLocalFileList(localFiles))
expect(result.current.startRunBtnDisabled).toBe(false)
})
it('waits for draft synchronization and exposes pending state until the run settles', async () => {
const pending = createDeferred<WorkflowRunNodeExecutionResponse>()
mocks.run.mockReturnValue(pending.promise)
mocks.sync.mockImplementation(async () => undefined)
const { result, props, nodeData } = setup(createProps(), { localFileList: localFiles })
act(() => result.current.handleRunWithSyncDraft())
expect(mocks.run).not.toHaveBeenCalled()
expect(nodeData()._singleRunningStatus).toBe(NodeRunningStatus.Running)
act(() => mocks.sync.mock.calls[0]![1]?.onSuccess?.())
await waitFor(() => expect(result.current.isPending).toBe(true))
await act(async () => pending.resolve({ id: 'run-1', status: NodeRunningStatus.Succeeded }))
await waitFor(() => expect(props.onSuccess).toHaveBeenCalledOnce())
expect(result.current.isPending).toBe(false)
})
it('marks a rejected request failed, invalidates last run and refreshes default error outputs', async () => {
mocks.run.mockRejectedValue(new Error('Run failed'))
const { result, props, client, lastRunKey, nodeData } = setup(createProps(), {
localFileList: localFiles,
})
act(() => result.current.handleRunWithSyncDraft())
await waitFor(() => expect(props.appendNodeInspectVars).toHaveBeenCalledOnce())
expect(client.getQueryState(lastRunKey)?.isInvalidated).toBe(true)
expect(nodeData()).toMatchObject({
_singleRunningStatus: NodeRunningStatus.Failed,
_isSingleRun: false,
})
expect(props.onSuccess).not.toHaveBeenCalled()
})
it.each(['succeeded', 'failed'] as const)(
'ignores a %s completion after the user pauses',
async (outcome) => {
const pending = createDeferred<WorkflowRunNodeExecutionResponse>()
mocks.run.mockReturnValue(pending.promise)
const { result, props, rerender, client, lastRunKey, nodeData } = setup(createProps(), {
localFileList: localFiles,
})
act(() => result.current.handleRunWithSyncDraft())
await waitFor(() => expect(result.current.isPending).toBe(true))
rerender({ ...props, isPaused: true })
await act(async () => {
if (outcome === 'failed') pending.reject(new Error('Run failed'))
else pending.resolve({ id: 'run-1', status: NodeRunningStatus.Succeeded })
})
await waitFor(() => expect(result.current.isPending).toBe(false))
expect(nodeData()._singleRunningStatus).toBe(NodeRunningStatus.Running)
expect(mocks.inspect).not.toHaveBeenCalled()
expect(props.onSuccess).not.toHaveBeenCalled()
expect(client.getQueryState(lastRunKey)?.isInvalidated).toBe(outcome === 'failed')
},
)
it('sets the chained run result without refreshing inspector variables', async () => {
const { result, props } = setup(
createProps({
isRunAfterSingleRun: true,
payload: createData({ _singleRunningStatus: NodeRunningStatus.Running }),
}),
{ localFileList: localFiles },
)
act(() => result.current.handleRunWithSyncDraft())
await waitFor(() =>
expect(props.setRunResult).toHaveBeenCalledWith(
expect.objectContaining({ id: 'run-1', status: NodeRunningStatus.Succeeded }),
),
)
expect(mocks.inspect).not.toHaveBeenCalled()
expect(props.onSuccess).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,71 @@
import type { DataSourceNodeType } from '../../types'
import { act } from '@testing-library/react'
import { createDatasourceProvider } from '@/app/components/rag-pipeline/__tests__/datasource-fixtures'
import { VarKindType } from '@/app/components/workflow/nodes/_base/types'
import { BlockEnum } from '@/app/components/workflow/types'
import { renderNodeConfig } from '../../../__tests__/render-node-config'
import { useConfig } from '../use-config'
const data: DataSourceNodeType = {
title: 'Datasource',
desc: '',
type: BlockEnum.DataSource,
plugin_id: 'langgenius/file',
provider_type: 'local_file',
provider_name: 'file',
datasource_name: 'local-file',
datasource_label: 'File',
datasource_parameters: {},
datasource_configurations: {},
_dataSourceStartToAdd: true,
}
it('persists initialization, extensions and consecutive parameter edits in the datasource graph', () => {
const { result } = renderNodeConfig(useConfig, data)
expect(result.current.flow.getNode('node')?.data._dataSourceStartToAdd).toBe(false)
act(() => result.current.config.handleFileExtensionsChange(['pdf', 'csv']))
act(() =>
result.current.config.handleParametersChange({
count: { type: VarKindType.constant, value: 0 },
}),
)
expect(result.current.flow.getNode('node')?.data).toMatchObject({
_dataSourceStartToAdd: false,
fileExtensions: ['pdf', 'csv'],
datasource_parameters: { count: { type: VarKindType.constant, value: 0 } },
})
})
it('reads generated datasource output declarations and safely keeps unknown property schemas visible', () => {
const provider = createDatasourceProvider()
provider.declaration.datasources![0]!.output_schema = {
properties: {
metadata: { type: 'object', properties: { title: { type: 'string' } } },
names: { type: 'array', items: { type: 'string' }, description: 'Names' },
missingItems: { type: 'array' },
unrestricted: true,
forbidden: false,
empty: {},
nullable: null,
},
}
const { result } = renderNodeConfig((id, payload) => useConfig(id, payload, [provider]), data)
expect(result.current.config.hasObjectOutput).toBe(true)
expect(result.current.config.outputSchema.map(({ name, type }) => ({ name, type }))).toEqual([
{ name: 'metadata', type: 'object' },
{ name: 'names', type: 'array[string]' },
{ name: 'missingItems', type: 'array[unknown]' },
{ name: 'unrestricted', type: 'unknown' },
{ name: 'forbidden', type: 'unknown' },
{ name: 'empty', type: 'unknown' },
{ name: 'nullable', type: 'unknown' },
])
})
it('accepts a datasource without declared outputs', () => {
const { result } = renderNodeConfig(
(id, payload) => useConfig(id, payload, [createDatasourceProvider()]),
data,
)
expect(result.current.config.outputSchema).toEqual([])
expect(result.current.config.hasObjectOutput).toBe(false)
})
@@ -0,0 +1,221 @@
import type { CustomRunFormProps, DataSourceNodeType } from '../types'
import type { NodeTracing } from '@/types/workflow'
import { useMutation } from '@tanstack/react-query'
import { useEffect, useMemo, useRef } from 'react'
import { useStoreApi } from 'reactflow'
import { useShallow } from 'zustand/react/shallow'
import {
useDataSourceStore,
useDataSourceStoreWithSelector,
} from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
import { selectDatasourceSingleRun } from '@/app/components/rag-pipeline/utils/query-select'
import { DatasourceType } from '@/models/pipeline'
import { consoleQuery } from '@/service/console'
import { useInvalidateLastRun } from '@/service/use-workflow'
import { fetchNodeInspectVars } from '@/service/workflow'
import { TransferMethod } from '@/types/app'
import { useNodeDataUpdate } from '../../../hooks/use-node-data-update'
import { useNodesSyncDraft } from '../../../hooks/use-nodes-sync-draft'
import { NodeRunningStatus } from '../../../types'
const useBeforeRunForm = ({
nodeId,
resource,
payload,
setRunResult,
isPaused,
isRunAfterSingleRun,
setIsRunAfterSingleRun,
onSuccess,
appendNodeInspectVars,
}: CustomRunFormProps) => {
if (resource.kind !== 'knowledge-pipeline')
throw new Error('Data source runs require a knowledge pipeline')
const flowId = resource.pipelineId
const store = useStoreApi()
const dataSourceStore = useDataSourceStore()
const isPausedRef = useRef(isPaused)
const { handleNodeDataUpdate } = useNodeDataUpdate()
const datasourceType = payload.provider_type as DatasourceType
const datasourceNodeData = payload as DataSourceNodeType
const { localFileList, onlineDocuments, websitePages, selectedFileIds } =
useDataSourceStoreWithSelector(
useShallow((state) => ({
localFileList: state.localFileList,
onlineDocuments: state.onlineDocuments,
websitePages: state.websitePages,
selectedFileIds: state.selectedFileIds,
})),
)
const startRunBtnDisabled = useMemo(() => {
if (!datasourceNodeData) return false
if (datasourceType === DatasourceType.localFile)
return !localFileList.length || localFileList.some((file) => !file.file.id)
if (datasourceType === DatasourceType.onlineDocument) return !onlineDocuments.length
if (datasourceType === DatasourceType.websiteCrawl) return !websitePages.length
if (datasourceType === DatasourceType.onlineDrive) return !selectedFileIds.length
return false
}, [
datasourceNodeData,
datasourceType,
localFileList,
onlineDocuments.length,
selectedFileIds.length,
websitePages.length,
])
useEffect(() => {
isPausedRef.current = isPaused
}, [isPaused])
const runningStatus = payload._singleRunningStatus || NodeRunningStatus.NotStart
const setNodeRunning = () => {
handleNodeDataUpdate({
id: nodeId,
data: {
...payload,
_singleRunningStatus: NodeRunningStatus.Running,
},
})
}
const invalidLastRun = useInvalidateLastRun(resource, nodeId)
const updateRunResult = async (data: NodeTracing | undefined) => {
const isPaused = isPausedRef.current
// The backend don't support pause the single run, so the frontend handle the pause state.
if (isPaused) return
const canRunLastRun = !isRunAfterSingleRun || runningStatus === NodeRunningStatus.Succeeded
if (!canRunLastRun) {
setRunResult(data ?? null)
return
}
// run fail may also update the inspect vars when the node set the error default output.
const vars = await fetchNodeInspectVars(resource, nodeId)
const { getNodes } = store.getState()
const nodes = getNodes()
appendNodeInspectVars(nodeId, vars, nodes)
if (data?.status === NodeRunningStatus.Succeeded) onSuccess()
}
const { mutate: handleDatasourceSingleRun, isPending } = useMutation(
consoleQuery.rag.pipelines.byPipelineId.workflows.draft.datasource.variablesInspect.post.mutationOptions(),
)
const handleRun = () => {
let datasourceInfo: Record<string, any> = {}
const { currentCredentialId: credentialId } = dataSourceStore.getState()
if (datasourceType === DatasourceType.localFile) {
const { localFileList } = dataSourceStore.getState()
const { id, name, type, size, extension, mime_type } = localFileList[0]!.file
const documentInfo = {
related_id: id,
name,
type,
size,
extension,
mime_type,
url: '',
transfer_method: TransferMethod.local_file,
}
datasourceInfo = documentInfo
}
if (datasourceType === DatasourceType.onlineDocument) {
const { onlineDocuments } = dataSourceStore.getState()
const { workspace_id, ...rest } = onlineDocuments[0]!
const documentInfo = {
workspace_id,
page: rest,
credential_id: credentialId,
}
datasourceInfo = documentInfo
}
if (datasourceType === DatasourceType.websiteCrawl) {
const { websitePages } = dataSourceStore.getState()
datasourceInfo = {
...websitePages[0],
credential_id: credentialId,
}
}
if (datasourceType === DatasourceType.onlineDrive) {
const { bucket, onlineDriveFileList, selectedFileIds } = dataSourceStore.getState()
const file = onlineDriveFileList.find((file) => file.id === selectedFileIds[0])
datasourceInfo = {
bucket,
id: file?.id,
type: file?.type,
credential_id: credentialId,
}
}
let hasError = false
handleDatasourceSingleRun(
{
params: { pipeline_id: flowId },
body: {
start_node_id: nodeId,
start_node_title: datasourceNodeData.title,
datasource_type: datasourceType,
datasource_info: datasourceInfo,
},
},
{
onError: () => {
hasError = true
invalidLastRun()
if (isPausedRef.current) return
handleNodeDataUpdate({
id: nodeId,
data: {
...payload,
_isSingleRun: false,
_singleRunningStatus: NodeRunningStatus.Failed,
},
})
},
onSettled: (data) => {
// A failed request settles without data; still refresh inspect vars.
updateRunResult(data ? selectDatasourceSingleRun(data) : undefined)
if (!hasError && !isPausedRef.current) {
handleNodeDataUpdate({
id: nodeId,
data: {
...payload,
_isSingleRun: false,
_singleRunningStatus: NodeRunningStatus.Succeeded,
},
})
}
},
},
)
}
const { handleSyncWorkflowDraft } = useNodesSyncDraft()
const handleRunWithSyncDraft = () => {
setNodeRunning()
setIsRunAfterSingleRun(true)
handleSyncWorkflowDraft(true, true, {
onSuccess() {
handleRun()
},
})
}
return {
isPending,
handleRunWithSyncDraft,
datasourceType,
datasourceNodeData,
startRunBtnDisabled,
}
}
export default useBeforeRunForm
@@ -0,0 +1,114 @@
import type { RagPipelineDatasourceProviderResponse } from '@dify/contracts/api/console/rag/types.gen'
import type { ResourceVarInputs } from '../../_base/types'
import type { DataSourceNodeType } from '../types'
import { useCallback, useEffect, useMemo } from 'react'
import { useReactFlow } from 'reactflow'
import { matchDataSource } from '@/app/components/workflow/utils/plugin-install-check'
import { useNodeDataUpdate } from '../../../hooks/use-node-data-update'
export const useConfig = (
id: string,
data: DataSourceNodeType,
dataSourceList?: RagPipelineDatasourceProviderResponse[],
) => {
const { getNodes } = useReactFlow<DataSourceNodeType>()
const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate()
const getNodeData = useCallback(() => {
const nodes = getNodes()
return nodes.find((node) => node.id === id)
}, [getNodes, id])
const handleNodeDataUpdate = useCallback(
(data: Partial<DataSourceNodeType>) => {
handleNodeDataUpdateWithSyncDraft({
id,
data,
})
},
[id, handleNodeDataUpdateWithSyncDraft],
)
const handleLocalFileDataSourceInit = useCallback(() => {
const nodeData = getNodeData()
if (nodeData?.data._dataSourceStartToAdd && nodeData?.data.provider_type === 'local_file') {
handleNodeDataUpdate({
...nodeData.data,
_dataSourceStartToAdd: false,
})
}
}, [getNodeData, handleNodeDataUpdate])
useEffect(() => {
handleLocalFileDataSourceInit()
}, [handleLocalFileDataSourceInit])
const handleFileExtensionsChange = useCallback(
(fileExtensions: string[]) => {
const nodeData = getNodeData()
handleNodeDataUpdate({
...nodeData?.data,
fileExtensions,
})
},
[handleNodeDataUpdate, getNodeData],
)
const handleParametersChange = useCallback(
(datasource_parameters: ResourceVarInputs) => {
const nodeData = getNodeData()
handleNodeDataUpdate({
...nodeData?.data,
datasource_parameters,
})
},
[handleNodeDataUpdate, getNodeData],
)
const outputSchema = useMemo(() => {
if (!dataSourceList) return []
const provider = matchDataSource(dataSourceList, data)
const datasource = provider?.declaration.datasources?.find(
(item) => item.identity.name === data.datasource_name,
)
const properties = datasource?.output_schema?.properties
if (!properties || typeof properties !== 'object' || Array.isArray(properties)) return []
return Object.entries(properties).map(([name, value]: [string, unknown]) => {
const schema = value && typeof value === 'object' && !Array.isArray(value) ? value : {}
const type =
'type' in schema && typeof schema.type === 'string' && schema.type ? schema.type : 'unknown'
const items = 'items' in schema ? schema.items : undefined
const itemType =
items &&
typeof items === 'object' &&
'type' in items &&
typeof items.type === 'string' &&
items.type
? items.type
: 'unknown'
return {
name,
value,
type: type === 'array' ? `array[${itemType}]` : type,
description:
'description' in schema && typeof schema.description === 'string'
? schema.description
: '',
isObject: type === 'object',
}
})
}, [data, dataSourceList])
const hasObjectOutput = outputSchema.some((output) => output.isObject)
return {
handleFileExtensionsChange,
handleParametersChange,
outputSchema,
hasObjectOutput,
}
}
@@ -0,0 +1,31 @@
import type { FC } from 'react'
import type { DataSourceNodeType } from './types'
import type { NodeProps } from '@/app/components/workflow/types'
import { memo } from 'react'
import { InstallPluginButton } from '@/app/components/workflow/nodes/_base/components/install-plugin-button'
import { useNodePluginInstallation } from '../../hooks/use-node-plugin-installation'
const Node: FC<NodeProps<DataSourceNodeType>> = ({ data }) => {
const { isChecking, isMissing, uniqueIdentifier, canInstall, onInstallSuccess } =
useNodePluginInstallation(data)
const showInstallButton = !isChecking && isMissing && canInstall && uniqueIdentifier
if (!showInstallButton) return null
return (
<div className="relative mb-1 px-3 py-1">
<div className="pointer-events-auto absolute -top-8 right-3 z-40">
<InstallPluginButton
size="small"
extraIdentifiers={[data.plugin_id, data.provider_name].filter(Boolean) as string[]}
className="font-medium! text-text-accent!"
uniqueIdentifier={uniqueIdentifier!}
onSuccess={onInstallSuccess}
/>
</div>
</div>
)
}
export default memo(Node)
@@ -0,0 +1,155 @@
import type { FC } from 'react'
import type { DataSourceNodeType } from './types'
import type { NodePanelProps } from '@/app/components/workflow/types'
import { useAtomValue } from 'jotai'
import { memo, useMemo } from 'react'
import { useTranslation } from 'react-i18next'
import TagInput from '@/app/components/base/tag-input'
import { editorResourceKindAtom } from '@/app/components/workflow/editor-resource/state'
import { BoxGroupField } from '@/app/components/workflow/nodes/_base/components/layout'
import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
import StructureOutputItem from '@/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/show'
import { useStore } from '@/app/components/workflow/store'
import { datasourceParametersToFormSchemas } from '@/app/components/workflow/utils/data-source'
import { matchDataSource } from '@/app/components/workflow/utils/plugin-install-check'
import { useNodesReadOnly } from '../../hooks/use-workflow'
import useMatchSchemaType, {
getMatchedSchemaType,
} from '../_base/components/variable/use-match-schema-type'
import ToolForm from '../tool/components/tool-form'
import { COMMON_OUTPUT, LOCAL_FILE_OUTPUT } from './constants'
import { useConfig } from './hooks/use-config'
import { DataSourceClassification } from './types'
const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
const { t } = useTranslation(['workflow'] as const)
const { nodesReadOnly } = useNodesReadOnly()
const dataSourceList = useStore((s) => s.dataSourceList)
const { provider_type, fileExtensions = [], datasource_parameters } = data
const { handleFileExtensionsChange, handleParametersChange, outputSchema, hasObjectOutput } =
useConfig(id, data, dataSourceList)
const isLocalFile = provider_type === DataSourceClassification.localFile
const currentDataSource = matchDataSource(dataSourceList ?? [], data)
const currentDataSourceItem = currentDataSource?.declaration.datasources?.find(
(item) => item.identity.name === data.datasource_name,
)
const formSchemas = useMemo(
() => datasourceParametersToFormSchemas(currentDataSourceItem?.parameters),
[currentDataSourceItem],
)
const resourceKind = useAtomValue(editorResourceKindAtom)
const setShowInputFieldPanel = useStore((s) => s.setShowInputFieldPanel)
const { schemaTypeDefinitions } = useMatchSchemaType()
return (
<div>
{currentDataSource?.is_authorized && !isLocalFile && !!formSchemas?.length && (
<BoxGroupField
boxGroupProps={{
boxProps: { withBorderBottom: true },
}}
fieldProps={{
fieldTitleProps: {
title: t(($) => $['nodes.tool.inputVars'], { ns: 'workflow' }),
},
supportCollapse: true,
}}
>
{formSchemas.length > 0 && (
<ToolForm
readOnly={nodesReadOnly}
nodeId={id}
schema={formSchemas}
staticSchema
value={datasource_parameters}
onChange={handleParametersChange}
showManageInputField={resourceKind === 'knowledge-pipeline'}
onManageInputField={() => setShowInputFieldPanel?.(true)}
/>
)}
</BoxGroupField>
)}
{isLocalFile && (
<BoxGroupField
boxGroupProps={{
boxProps: { withBorderBottom: true },
}}
fieldProps={{
fieldTitleProps: {
title: t(($) => $['nodes.dataSource.supportedFileFormats'], { ns: 'workflow' }),
},
}}
>
<div className="rounded-lg bg-components-input-bg-normal p-1 pt-0">
<TagInput
items={fileExtensions}
onChange={handleFileExtensionsChange}
placeholder={t(($) => $['nodes.dataSource.supportedFileFormatsPlaceholder'], {
ns: 'workflow',
})}
inputClassName="bg-transparent"
disableAdd={nodesReadOnly}
disableRemove={nodesReadOnly}
/>
</div>
</BoxGroupField>
)}
<OutputVars>
{COMMON_OUTPUT.map((item, index) => (
<VarItem
key={index}
name={item.name}
type={item.type}
description={item.description}
isIndent={hasObjectOutput}
/>
))}
{isLocalFile &&
LOCAL_FILE_OUTPUT.map((item, index) => (
<VarItem
key={index}
name={item.name}
type={item.type}
description={item.description}
subItems={item.subItems.map((item) => ({
name: item.name,
type: item.type,
description: item.description,
}))}
/>
))}
{outputSchema.map((outputItem) => {
const schema =
outputItem.value &&
typeof outputItem.value === 'object' &&
!Array.isArray(outputItem.value)
? outputItem.value
: {}
const schemaType = getMatchedSchemaType(schema, schemaTypeDefinitions)
return (
<div key={outputItem.name}>
{outputItem.isObject ? (
<StructureOutputItem
rootClassName="code-sm-semibold text-text-secondary"
payload={{
schema: { properties: { [outputItem.name]: { ...schema, schemaType } } },
}}
/>
) : (
<VarItem
name={outputItem.name}
type={`${outputItem.type.toLocaleLowerCase()}${schemaType ? ` (${schemaType})` : ''}`}
description={outputItem.description}
isIndent={hasObjectOutput}
/>
)}
</div>
)
})}
</OutputVars>
</div>
)
}
export default memo(Panel)
@@ -0,0 +1,35 @@
import type { DatasourceProviderType } from '@dify/contracts/api/console/workspaces/types.gen'
import type { Dispatch, SetStateAction } from 'react'
import type { ResourceVarInputs } from '../_base/types'
import type { EditorResource } from '@/app/components/workflow/editor-resource/resource'
import type { CommonNodeType, Node } from '@/app/components/workflow/types'
import type { NodeTracing, VarInInspect } from '@/types/workflow'
export const DataSourceClassification = {
localFile: 'local_file',
} as const
export type DataSourceNodeType = CommonNodeType & {
fileExtensions?: string[]
plugin_id: string
provider_type: DatasourceProviderType
provider_name: string
datasource_name: string
datasource_label: string
datasource_parameters: ResourceVarInputs
datasource_configurations: Record<string, any>
plugin_unique_identifier?: string
}
export type CustomRunFormProps = {
nodeId: string
resource: EditorResource
payload: CommonNodeType
setRunResult: Dispatch<SetStateAction<NodeTracing | null>>
setIsRunAfterSingleRun: Dispatch<SetStateAction<boolean>>
isPaused: boolean
isRunAfterSingleRun: boolean
onSuccess: () => void
onCancel: () => void
appendNodeInspectVars: (nodeId: string, vars: VarInInspect[], nodes: Node[]) => void
}
@@ -0,0 +1,15 @@
import { VarType } from '@/app/components/workflow/types'
import { PipelineInputVarType } from '@/models/pipeline'
export const inputVarTypeToVarType = (type: PipelineInputVarType): VarType => {
return (
(
{
[PipelineInputVarType.number]: VarType.number,
[PipelineInputVarType.singleFile]: VarType.file,
[PipelineInputVarType.multiFiles]: VarType.arrayFile,
[PipelineInputVarType.checkbox]: VarType.boolean,
} as any
)[type] || VarType.string
)
}