import json import logging import os import zipfile from flask import request, send_file from flask_restful import Resource, marshal, marshal_with, reqparse from werkzeug.exceptions import Forbidden, NotFound import services from controllers.console import api from controllers.console.error import FileTooLargeError, UnsupportedFileTypeError from controllers.console.wraps import account_initialization_required, setup_required from extensions.ext_storage import storage from fields.intention_fields import ( intention_corpus_detail_fields, intention_corpus_similarity_question_fields, intention_detail_fields, intention_keyword_detail_fields, intention_keyword_fields, intention_page_fields, intention_train_file_binding_fields, intention_train_file_fields, intention_train_task_fields, intention_type_detail_fields, intention_type_page_fields, ) from libs.login import current_user, login_required from models import UploadFile from models.intention import IntentionTrainFile, IntentionTrainTask from services.errors.intention import IntentionTrainFileDuplicateError from services.file_service import FileService from services.intention_service import ( IntentionCorpusService, IntentionCorpusSimilarityQuestionService, IntentionKeywordService, IntentionService, IntentionTrainFileBindingService, IntentionTrainFileService, IntentionTrainTaskService, IntentionTypeService, ) from services.upload_file_service import UploadFileService class IntentionListApi(Resource): @setup_required @login_required @account_initialization_required def get(self): page = request.args.get("page", default=1, type=int) limit = request.args.get("limit", default=20, type=int) type_id = request.args.get("type_id", default=None, type=str) name_search = request.args.get("name_search", default=None, type=str) intentions, total = IntentionService.get_intentions(page, limit, type_id, name_search) data = marshal(intentions, intention_page_fields) response = {"data": data, "has_more": len(intentions) == limit, "limit": limit, "total": total, "page": page} return response, 200 @setup_required @login_required @account_initialization_required def post(self): parser = reqparse.RequestParser() parser.add_argument( "name", nullable=False, required=True, help="type is required. Name must be between 1 to 40 characters.", ) parser.add_argument( "type_id", nullable=False, required=True, help="type is required.", ) args = parser.parse_args() intention = IntentionService.save_intention(args) response = marshal(intention, intention_detail_fields) return response, 200 class IntentionApi(Resource): @setup_required @login_required @account_initialization_required def get(self, intention_id): intention = IntentionService.get_intention(intention_id) return marshal(intention, intention_detail_fields) @setup_required @login_required @account_initialization_required def patch(self, intention_id): parser = reqparse.RequestParser() parser.add_argument( "name", nullable=False, required=True, help="type is required. Name must be between 1 to 40 characters.", ) parser.add_argument( "type_id", nullable=False, required=True, help="type is required.", ) args = parser.parse_args() intention = IntentionService.update_intention(intention_id, args) response = marshal(intention, intention_detail_fields) return response, 200 @setup_required @login_required @account_initialization_required def delete(self, intention_id): IntentionService.delete_intention(intention_id) return 200 class IntentionTypeListApi(Resource): @setup_required @login_required @account_initialization_required def get(self): page = request.args.get("page", default=1, type=int) limit = request.args.get("limit", default=20, type=int) search = request.args.get("search", default=None, type=str) intention_types, total = IntentionTypeService.get_intention_types(page, limit, search) data = marshal(intention_types, intention_type_page_fields) response = {"data": data, "has_more": len(intention_types) == limit, "limit": limit, "total": total, "page": page} return response, 200 @setup_required @login_required @account_initialization_required def post(self): parser = reqparse.RequestParser() parser.add_argument( "name", nullable=False, required=True, help="type is required. Name must be between 1 to 40 characters.", ) args = parser.parse_args() intention_type = IntentionTypeService.save_intention_type(args) response = marshal(intention_type, intention_type_detail_fields) return response, 200 class IntentionTypeApi(Resource): @setup_required @login_required @account_initialization_required def get(self, intention_type_id): intention_type = IntentionTypeService.get_intention_type(intention_type_id) return marshal(intention_type, intention_type_detail_fields) @setup_required @login_required @account_initialization_required def patch(self, intention_type_id): parser = reqparse.RequestParser() parser.add_argument( "name", nullable=False, required=True, help="type is required. Name must be between 1 to 40 characters.", ) args = parser.parse_args() intention_type = IntentionTypeService.update_intention_type(intention_type_id, args) return marshal(intention_type, intention_type_detail_fields), 200 @setup_required @login_required @account_initialization_required def delete(self, intention_type_id): IntentionTypeService.delete_intention_type(intention_type_id) return 200 class IntentionKeywordListApi(Resource): @setup_required @login_required @account_initialization_required @marshal_with(intention_keyword_fields) def get(self, intention_id): search = request.args.get("search", default=None, type=str) intention = IntentionService.get_intention(intention_id) if not intention: raise NotFound("Intention not found") intention_keywords = IntentionKeywordService.get_intention_keywords(intention_id, search) return intention_keywords, 200 @setup_required @login_required @account_initialization_required @marshal_with(intention_keyword_detail_fields) def post(self, intention_id): parser = reqparse.RequestParser() parser.add_argument( "name", nullable=False, required=True, help="type is required. Name must be between 1 to 40 characters.", ) args = parser.parse_args() intention = IntentionService.get_intention(intention_id) if not intention: raise NotFound("Intention not found") intention_keyword = IntentionKeywordService.save_intention_keyword(intention_id, args) return intention_keyword, 200 @setup_required @login_required @account_initialization_required def delete(self, intention_id): intention = IntentionService.get_intention(intention_id) if not intention: raise NotFound("Intention not found") IntentionKeywordService.delete_intention_keywords_by_intention_id(intention_id) return 200 class IntentionKeywordApi(Resource): @setup_required @login_required @account_initialization_required def get(self, intention_keyword_id): intention_keyword = IntentionKeywordService.get_intention_keyword(intention_keyword_id) if not intention_keyword: return {}, 200 return marshal(intention_keyword, intention_keyword_detail_fields), 200 @setup_required @login_required @account_initialization_required @marshal_with(intention_keyword_detail_fields) def patch(self, intention_keyword_id): parser = reqparse.RequestParser() parser.add_argument( "name", nullable=False, required=True, help="type is required. Name must be between 1 to 40 characters.", ) parser.add_argument( "intention_id", nullable=False, required=True, help="type is required.", ) args = parser.parse_args() intention_keyword = IntentionKeywordService.update_intention_keyword(intention_keyword_id, args) return intention_keyword, 200 @setup_required @login_required @account_initialization_required def delete(self, intention_keyword_id): IntentionKeywordService.delete_intention_keyword(intention_keyword_id) return 200 class IntentionKeywordBatchApi(Resource): @setup_required @login_required @account_initialization_required def post(self): parser = reqparse.RequestParser() parser.add_argument( "method", nullable=False, required=True, help="method is required.", choices=["create", "update", "delete"], type=str, location="json", ) parser.add_argument( "delete_data", nullable=False, required=True, help="delete_data is required.", type=list, location="json", ) args = parser.parse_args() logging.info(args) method = args["method"] if method == "delete": intention_keyword_ids = args["delete_data"] IntentionKeywordService.delete_intention_keywords(intention_keyword_ids) return 200 else: raise NotFound(f"method with name {method} not found") class IntentionCorpusListApi(Resource): @setup_required @login_required @account_initialization_required def get(self): page = request.args.get("page", default=1, type=int) limit = request.args.get("limit", default=20, type=int) question_search = request.args.get("question_search", default=None, type=str) intention_id = request.args.get("intention_id", default=None, type=str) intention_corpus, total = IntentionCorpusService.get_page_intention_corpus( page, limit, question_search, intention_id) data = marshal(intention_corpus, intention_corpus_detail_fields) response = {"data": data, "has_more": len(intention_corpus) == limit, "limit": limit, "total": total, "page": page} return response, 200 @setup_required @login_required @account_initialization_required def post(self): parser = reqparse.RequestParser() parser.add_argument( "question", nullable=False, required=True, help="type is required. Question must be between 1 to 40 characters.", ) parser.add_argument( "question_config", nullable=True, required=False, location="json", ) parser.add_argument( "intention_id", nullable=False, required=True, help="type is required.", ) args = parser.parse_args() intention_corpus = IntentionCorpusService.save_intention_corpus(args) return marshal(intention_corpus, intention_corpus_detail_fields), 200 class IntentionCorpusApi(Resource): @setup_required @login_required @account_initialization_required def get(self, corpus_id): intention_corpus = IntentionCorpusService.get_intention_corpus(corpus_id) if not intention_corpus: raise NotFound(f"IntentionCorpus with id {corpus_id} not found") return marshal(intention_corpus, intention_corpus_detail_fields), 200 @setup_required @login_required @account_initialization_required def patch(self, corpus_id): parser = reqparse.RequestParser() parser.add_argument( "question", nullable=True, required=False, type=str, location="json", ) parser.add_argument( "question_config", nullable=True, required=False, location="json", ) parser.add_argument( "intention_id", nullable=True, required=False, type=str, location="json", ) args = parser.parse_args() intention_corpus = IntentionCorpusService.update_intention_corpus(corpus_id, args) return marshal(intention_corpus, intention_corpus_detail_fields), 200 @setup_required @login_required @account_initialization_required def delete(self, corpus_id): intention_corpus = IntentionCorpusService.get_intention_corpus(corpus_id) if not intention_corpus: raise NotFound(f"未发现Id未{corpus_id}的训练语料") similarity_questions = intention_corpus.similarity_questions if similarity_questions: raise Forbidden(f"存在与其关联的相似问题,无法删除Id为{corpus_id}训练语料") IntentionCorpusService.delete_intention_corpus(intention_corpus) return 200 class IntentionCorpusSimilarityQuestionApi(Resource): @setup_required @login_required @account_initialization_required def get(self, corpus_id): search = request.args.get("search", default=None, type=str) similarity_questions = ( IntentionCorpusSimilarityQuestionService .get_similarity_questions_by_corpus_id_like_question(corpus_id, search) ) return marshal(similarity_questions, intention_corpus_similarity_question_fields), 200 @setup_required @login_required @account_initialization_required def post(self, corpus_id): parser = reqparse.RequestParser() parser.add_argument( "question", nullable=False, required=True, help="type is required. Question must be between 1 to 40 characters.", location="json", ) parser.add_argument( "question_config", nullable=True, required=False, location="json", ) args = parser.parse_args() intention_corpus_similarity_question = ( IntentionCorpusSimilarityQuestionService.save_similarity_question(corpus_id, args) ) return marshal(intention_corpus_similarity_question, intention_corpus_similarity_question_fields), 200 @setup_required @login_required @account_initialization_required def delete(self, corpus_id): IntentionCorpusSimilarityQuestionService.delete_similarity_question_by_corpus_id(corpus_id) return 200 class IntentionCorpusSimilarityQuestionUpdateAndDeleteApi(Resource): @setup_required @login_required @account_initialization_required def patch(self, similarity_question_id): parser = reqparse.RequestParser() parser.add_argument( "question", nullable=True, required=False, help="type is required. Question must be between 1 to 40 characters.", location="json", ) parser.add_argument( "question_config", nullable=True, required=False, location="json", ) parser.add_argument( "corpus_id", nullable=True, required=False, location="json", ) args = parser.parse_args() similarity_question = ( IntentionCorpusSimilarityQuestionService.update_similarity_question(similarity_question_id, args) ) return marshal(similarity_question, intention_corpus_similarity_question_fields), 200 @setup_required @login_required @account_initialization_required def delete(self, similarity_question_id): IntentionCorpusSimilarityQuestionService.delete_similarity_question_by_id(similarity_question_id) return 200 class IntentionCorpusSimilarityQuestionBatchApi(Resource): @setup_required @login_required @account_initialization_required def post(self): parser = reqparse.RequestParser() parser.add_argument( "method", nullable=False, required=True, help="method is required.", choices=["create", "update", "delete"], type=str, location="json", ) parser.add_argument( "data", nullable=False, required=True, help="data is required.", type=list, location="json", ) args = parser.parse_args() logging.info(args) method = args["method"] if method == "delete": similarity_question_ids = args["data"] IntentionCorpusSimilarityQuestionService.delete_similarity_questions_by_ids(similarity_question_ids) return 200 else: raise NotFound(f"method with name {method} not found") class IntentionTrainTaskListApi(Resource): @setup_required @login_required @account_initialization_required def get(self): page = request.args.get("page", default=1, type=int) limit = request.args.get("limit", default=20, type=int) search = request.args.get("search", default=None, type=str) intention_train_tasks, total = IntentionTrainTaskService.get_page_intention_train_tasks( page, limit, search) data = marshal(intention_train_tasks, intention_train_task_fields) response = {"data": data, "has_more": len(intention_train_tasks) == limit, "limit": limit, "total": total, "page": page} return response, 200 @setup_required @login_required @account_initialization_required def post(self): parser = reqparse.RequestParser() parser.add_argument( "name", nullable=False, required=True, help="name is required.", location="json", ) parser.add_argument( "status", nullable=False, required=True, help="status is required.", choices=IntentionTrainTask.STATUS_LIST, location="json", ) args = parser.parse_args() train_task = IntentionTrainTaskService.save_train_task(args) return marshal(train_task, intention_train_task_fields), 200 class IntentionTrainTaskApi(Resource): @setup_required @login_required @account_initialization_required def patch(self, task_id): parser = reqparse.RequestParser() parser.add_argument( "name", nullable=False, required=True, help="name is required.", location="json", ) parser.add_argument( "status", nullable=False, required=True, help="status is required.", choices=IntentionTrainTask.STATUS_LIST, location="json", ) args = parser.parse_args() train_task = IntentionTrainTaskService.update_train_task(task_id, args) return marshal(train_task, intention_train_task_fields), 200 class IntentionTrainTaskDownloadApi(Resource): @setup_required @login_required @account_initialization_required def get(self, task_id): train_task = IntentionTrainTaskService.get_train_task(task_id) if train_task.status != "COMPLETED": raise Forbidden(f"Task with id {task_id} not completed") dataset_info: IntentionTrainFile = train_task.dataset_info model_info: IntentionTrainFile = train_task.model_info # 生成待下载的zip包 zip_filename = f"{train_task.name}.zip" with zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_DEFLATED) as zip_file: for train_file in [dataset_info, model_info]: source_info = json.loads(train_file.data_source_info) upload_file_id = source_info["upload_file_id"] upload_file: UploadFile = UploadFileService.get_upload_file(upload_file_id) storage.download(upload_file.key, upload_file.name) zip_file.write(upload_file.name, upload_file.name) os.remove(upload_file.name) # 下载zip包 response = send_file(zip_filename, as_attachment=True, download_name=zip_filename) # 清除临时文件 os.remove(zip_filename) return response class IntentionTrainFileApi(Resource): @setup_required @login_required @account_initialization_required def get(self): name = request.args.get("name", default=None, type=str) version = request.args.get("version", default=None, type=str) type = request.args.get("type", default=None, type=str) train_files = IntentionTrainFileService.get_train_files(name, version, type) return marshal(train_files, intention_train_file_fields), 200 @setup_required @login_required @account_initialization_required def post(self): name = request.form.get("name") version = request.form.get("version") type = request.form.get("type") train_file = IntentionTrainFileService.get_train_file(name, version, type) if train_file: raise IntentionTrainFileDuplicateError(f"IntentionTrainFile with name-version-type " f"{name}-{version}-{type} already exists.") data_source_type = request.form.get("data_source_type") # get file from request file = request.files["file"] filename = file.filename mimetype = file.mimetype if not filename or not mimetype: raise Forbidden("Invalid request.") try: upload_file = FileService.upload_file( filename=filename, content=file.read(), mimetype=mimetype, user=current_user, source=None, ) args = { "name": name, "version": version, "type": type, "data_source_type": data_source_type, "data_source_info": { "upload_file_id": upload_file.id } } intention_train_file = IntentionTrainFileService.save_train_file(args) return marshal(intention_train_file, intention_train_file_fields), 200 except services.errors.file.FileTooLargeError as file_too_large_error: raise FileTooLargeError(file_too_large_error.description) except services.errors.file.UnsupportedFileTypeError: raise UnsupportedFileTypeError() class IntentionTrainFileBindingApi(Resource): @setup_required @login_required @account_initialization_required def post(self): parser = reqparse.RequestParser() parser.add_argument( "file_id", nullable=False, required=True, help="file_id is required.", location="json", ) parser.add_argument( "task_id", nullable=False, required=True, help="task_id is required.", location="json", ) args = parser.parse_args() train_file_binding = IntentionTrainFileBindingService.save_train_file_binding(args) return marshal(train_file_binding, intention_train_file_binding_fields), 200 api.add_resource(IntentionListApi, "/intentions") api.add_resource(IntentionApi, "/intentions/") api.add_resource(IntentionTypeListApi, "/intentions/types") api.add_resource(IntentionTypeApi, "/intentions/types/") api.add_resource(IntentionKeywordListApi, "/intentions//keywords") api.add_resource(IntentionKeywordApi, "/intentions/keywords/") api.add_resource(IntentionKeywordBatchApi, "/intentions/keywords/batch") api.add_resource(IntentionCorpusListApi, "/intentions/corpus") api.add_resource(IntentionCorpusApi, "/intentions/corpus/") api.add_resource(IntentionCorpusSimilarityQuestionApi, "/intentions/corpus//similarity_questions") api.add_resource(IntentionCorpusSimilarityQuestionUpdateAndDeleteApi, "/intentions/similarity_questions/") api.add_resource(IntentionCorpusSimilarityQuestionBatchApi, "/intentions/similarity_questions/batch") api.add_resource(IntentionTrainTaskListApi, "/intentions/train_tasks") api.add_resource(IntentionTrainTaskApi, "/intentions/train_tasks/") api.add_resource(IntentionTrainTaskDownloadApi, "/intentions/train_tasks/download/") api.add_resource(IntentionTrainFileApi, "/intentions/train_files") api.add_resource(IntentionTrainFileBindingApi, "/intentions/train_file_bindings")