Feat:csv & docx support (#1139)

Co-authored-by: jyong <jyong@dify.ai>

api/controllers/console/datasets/file.py

@@ -26,7 +26,7 @@ from models.model import UploadFile
 
 cache = TTLCache(maxsize=None, ttl=30)
 
-ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx']
+ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx', 'docx', 'csv']
 PREVIEW_WORDS_LIMIT = 3000
 
 

web/app/components/datasets/create/embedding-process/index.module.css

@@ -89,7 +89,9 @@
 .fileIcon.csv {
   background-image: url(../assets/csv.svg);
 }
-
+.fileIcon.docx {
+  background-image: url(../assets/docx.svg);
+}
 .fileIcon.xlsx,
 .fileIcon.xls {
   background-image: url(../assets/xlsx.svg);

web/app/components/datasets/create/embedding-process/index.tsx

@@ -194,7 +194,7 @@ const EmbeddingProcess: FC<Props> = ({ datasetId, batchId, documents = [], index
       </div>
       <div className={s.progressContainer}>
         {indexingStatusBatchDetail.map(indexingStatusDetail => (
-          <div className={cn(
+          <div key={indexingStatusDetail.id} className={cn(
             s.sourceItem,
             indexingStatusDetail.indexing_status === 'error' && s.error,
             indexingStatusDetail.indexing_status === 'completed' && s.success,

web/app/components/datasets/create/file-uploader/index.module.css

@@ -98,7 +98,9 @@
 .fileIcon.csv {
   background-image: url(../assets/csv.svg);
 }
-
+.fileIcon.docx {
+  background-image: url(../assets/docx.svg);
+}
 .fileIcon.xlsx,
 .fileIcon.xls {
   background-image: url(../assets/xlsx.svg);

web/app/components/datasets/create/file-uploader/index.tsx

@@ -29,9 +29,9 @@ const ACCEPTS = [
   '.md',
   '.markdown',
   '.txt',
-  // '.xls',
   '.xlsx',
-  // '.csv',
+  '.csv',
+  '.docx',
 ]
 
 const FileUploader = ({
@@ -250,7 +250,6 @@ const FileUploader = ({
             className={cn(
               s.file,
               fileItem.progress < 100 && s.uploading,
-              // s.active,
             )}
           >
             {fileItem.progress < 100 && (
@@ -274,33 +273,6 @@ const FileUploader = ({
             </div>
           </div>
         ))}
-        {/* {currentFile && (
-          <div
-            // onClick={() => onPreview(currentFile)}
-            className={cn(
-              s.file,
-              uploading && s.uploading,
-              // s.active,
-            )}
-          >
-            {uploading && (
-              <div className={s.progressbar} style={{ width: `${percent}%` }}/>
-            )}
-            <div className={s.fileInfo}>
-              <div className={cn(s.fileIcon, s[getFileType(currentFile)])}/>
-              <div className={s.filename}>{currentFile.name}</div>
-              <div className={s.size}>{getFileSize(currentFile.size)}</div>
-            </div>
-            <div className={s.actionWrapper}>
-              {uploading && (
-                <div className={s.percent}>{`${percent}%`}</div>
-              )}
-              {!uploading && (
-                <div className={s.remove} onClick={() => removeFile(index)}/>
-              )}
-            </div>
-          </div>
-        )} */}
       </div>
     </div>
   )

web/app/components/datasets/create/step-one/index.tsx

@@ -65,7 +65,7 @@ const StepOne = ({
   const { dataset } = useDatasetDetailContext()
   const [showModal, setShowModal] = useState(false)
   const [currentFile, setCurrentFile] = useState<File | undefined>()
-  const [currentNotionPage, setCurrentNotionPage] = useState<Page | undefined>()
+  const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
   const { t } = useTranslation()
 
   const modalShowHandle = () => setShowModal(true)
@@ -78,7 +78,7 @@ const StepOne = ({
     setCurrentFile(undefined)
   }
 
-  const updateCurrentPage = (page: Page) => {
+  const updateCurrentPage = (page: NotionPage) => {
     setCurrentNotionPage(page)
   }
 

web/app/components/datasets/create/step-two/index.module.css

@@ -290,6 +290,10 @@
   background-image: url(../assets/csv.svg);
 }
 
+.fileIcon.docx {
+  background-image: url(../assets/docx.svg);
+}
+
 .fileIcon.xlsx,
 .fileIcon.xls {
   background-image: url(../assets/xlsx.svg);

web/app/components/datasets/documents/detail/completed/SegmentCard.tsx

@@ -43,7 +43,7 @@ type ISegmentCardProps = {
   scene?: UsageScene
   className?: string
   archived?: boolean
-  embeddingAvailable: boolean
+  embeddingAvailable?: boolean
 }
 
 const SegmentCard: FC<ISegmentCardProps> = ({

web/app/components/datasets/documents/style.module.css

@@ -87,6 +87,9 @@
 .csvIcon {
   background-image: url(~@/assets/csv.svg);
 }
+.docxIcon {
+  background-image: url(~@/assets/docx.svg);
+}
 .statusItemDetail {
   @apply h-8 font-medium border border-gray-200 inline-flex items-center rounded-lg pl-3 pr-4 mr-2;
 }

web/i18n/lang/dataset-creation.en.ts

@@ -23,7 +23,7 @@ const translation = {
       title: 'Upload text file',
       button: 'Drag and drop file, or',
       browse: 'Browse',
-      tip: 'Supports txt, html, markdown, xlsx, and pdf. Max {{size}}MB each.',
+      tip: 'Supports txt, html, markdown, xlsx, csv, docx and pdf. Max {{size}}MB each.',
       validation: {
         typeError: 'File type not supported',
         size: 'File too large. Maximum is {{size}}MB',

web/i18n/lang/dataset-creation.zh.ts

@@ -23,7 +23,7 @@ const translation = {
       title: '上传文本文件',
       button: '拖拽文件至此，或者',
       browse: '选择文件',
-      tip: '已支持 TXT、 HTML、 Markdown、 PDF、 XLSX，每个文件不超过 {{size}}MB。',
+      tip: '已支持 TXT、 HTML、 Markdown、 PDF、 XLSX、CSV、DOCX，每个文件不超过 {{size}}MB。',
       validation: {
         typeError: '文件类型不支持',
         size: '文件太大了，不能超过 {{size}}MB',