From 33a077d7fb5c5b3fa9d0196ffc8bc35085eb29a7 Mon Sep 17 00:00:00 2001 From: SherkeyXD <57581480+SherkeyXD@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:59:02 +0800 Subject: [PATCH 1/5] fix(ppocr): avoid overwriting detector settings in PPOCRv5/v6 and cleanup deprecated aliases --- .../fastdeploy_capi/vision/types_internal.h | 12 ------ .../fastdeploy/vision/ocr/ppocr/__init__.py | 20 ---------- fastdeploy/vision/ocr/ppocr/ppocr_v5.h | 31 ++------------ fastdeploy/vision/ocr/ppocr/ppocr_v6.h | 40 ++++++------------- 4 files changed, 15 insertions(+), 88 deletions(-) diff --git a/bindings/c_api/fastdeploy_capi/vision/types_internal.h b/bindings/c_api/fastdeploy_capi/vision/types_internal.h index f55b3c6e476..ec8aaf76605 100755 --- a/bindings/c_api/fastdeploy_capi/vision/types_internal.h +++ b/bindings/c_api/fastdeploy_capi/vision/types_internal.h @@ -191,12 +191,6 @@ DEFINE_PIPELINE_MODEL_WRAPPER_STRUCT(PPOCRv3, ppocrv3_model); // PPOCRv4 DEFINE_PIPELINE_MODEL_WRAPPER_STRUCT(PPOCRv4, ppocrv4_model); -// PPOCRv5 -DEFINE_PIPELINE_MODEL_WRAPPER_STRUCT(PPOCRv5, ppocrv5_model); - -// PPOCRv6 -DEFINE_PIPELINE_MODEL_WRAPPER_STRUCT(PPOCRv6, ppocrv6_model); - // PPStructureV2Table DEFINE_PIPELINE_MODEL_WRAPPER_STRUCT(PPStructureV2Table, ppstructurev2table_model); @@ -413,12 +407,6 @@ DECLARE_PIPELINE_MODEL_FUNC_FOR_GET_PTR_FROM_WRAPPER(PPOCRv3, fd_ppocrv3_wrapper // PPOCRv4 DECLARE_PIPELINE_MODEL_FUNC_FOR_GET_PTR_FROM_WRAPPER(PPOCRv4, fd_ppocrv4_wrapper); -// PPOCRv5 -DECLARE_PIPELINE_MODEL_FUNC_FOR_GET_PTR_FROM_WRAPPER(PPOCRv5, fd_ppocrv5_wrapper); - -// PPOCRv6 -DECLARE_PIPELINE_MODEL_FUNC_FOR_GET_PTR_FROM_WRAPPER(PPOCRv6, fd_ppocrv6_wrapper); - // PPStructureV2Table DECLARE_PIPELINE_MODEL_FUNC_FOR_GET_PTR_FROM_WRAPPER(PPStructureV2Table, fd_ppstructurev2_table_wrapper); diff --git a/bindings/python/fastdeploy/vision/ocr/ppocr/__init__.py b/bindings/python/fastdeploy/vision/ocr/ppocr/__init__.py index c26562cee8d..1af8943d5aa 100755 --- a/bindings/python/fastdeploy/vision/ocr/ppocr/__init__.py +++ b/bindings/python/fastdeploy/vision/ocr/ppocr/__init__.py @@ -937,16 +937,6 @@ def rec_batch_size(self, value): self.system_.rec_batch_size = value -class PPOCRSystemv6(PPOCRv6): - def __init__(self, det_model=None, cls_model=None, rec_model=None): - logging.warning( - "DEPRECATED: fd.vision.ocr.PPOCRSystemv6 is deprecated, " - "please use fd.vision.ocr.PPOCRv6 instead.") - super(PPOCRSystemv6, self).__init__(det_model, cls_model, rec_model) - - def predict(self, input_image): - return super(PPOCRSystemv6, self).predict(input_image) - class PPOCRv5(FastDeployModel): def __init__(self, det_model=None, cls_model=None, rec_model=None): @@ -1013,16 +1003,6 @@ def rec_batch_size(self, value): self.system_.rec_batch_size = value -class PPOCRSystemv5(PPOCRv5): - def __init__(self, det_model=None, cls_model=None, rec_model=None): - logging.warning( - "DEPRECATED: fd.vision.ocr.PPOCRSystemv5 is deprecated, " - "please use fd.vision.ocr.PPOCRv5 instead.") - super(PPOCRSystemv5, self).__init__(det_model, cls_model, rec_model) - - def predict(self, input_image): - return super(PPOCRSystemv5, self).predict(input_image) - class PPOCRv4(FastDeployModel): def __init__(self, det_model=None, cls_model=None, rec_model=None): diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v5.h b/fastdeploy/vision/ocr/ppocr/ppocr_v5.h index 6c9ecca4b4b..b0688787d06 100644 --- a/fastdeploy/vision/ocr/ppocr/ppocr_v5.h +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v5.h @@ -34,17 +34,7 @@ class FASTDEPLOY_DECL PPOCRv5 : public PPOCRv4 { PPOCRv5(fastdeploy::vision::ocr::DBDetector* det_model, fastdeploy::vision::ocr::Classifier* cls_model, fastdeploy::vision::ocr::Recognizer* rec_model) - : PPOCRv4(det_model, cls_model, rec_model) { - auto preprocess_shape = recognizer_->GetPreprocessor().GetRecImageShape(); - preprocess_shape[1] = 48; - recognizer_->GetPreprocessor().SetRecImageShape(preprocess_shape); - if (detector_ != nullptr) { - detector_->GetPostprocessor().SetDetDBThresh(0.3); - detector_->GetPostprocessor().SetDetDBBoxThresh(0.6); - detector_->GetPostprocessor().SetDetDBUnclipRatio(1.5); - detector_->GetPostprocessor().SetDetDBMaxCandidates(1000); - } - } + : PPOCRv4(det_model, cls_model, rec_model) {} /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively. * * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv5_det_infer @@ -52,17 +42,7 @@ class FASTDEPLOY_DECL PPOCRv5 : public PPOCRv4 { */ PPOCRv5(fastdeploy::vision::ocr::DBDetector* det_model, fastdeploy::vision::ocr::Recognizer* rec_model) - : PPOCRv4(det_model, rec_model) { - auto preprocess_shape = recognizer_->GetPreprocessor().GetRecImageShape(); - preprocess_shape[1] = 48; - recognizer_->GetPreprocessor().SetRecImageShape(preprocess_shape); - if (detector_ != nullptr) { - detector_->GetPostprocessor().SetDetDBThresh(0.3); - detector_->GetPostprocessor().SetDetDBBoxThresh(0.6); - detector_->GetPostprocessor().SetDetDBUnclipRatio(1.5); - detector_->GetPostprocessor().SetDetDBMaxCandidates(1000); - } - } + : PPOCRv4(det_model, rec_model) {} /** \brief Clone a new PPOCRv5 with less memory usage when multiple instances of the same model are created * @@ -81,10 +61,5 @@ class FASTDEPLOY_DECL PPOCRv5 : public PPOCRv4 { } // namespace pipeline -namespace application { -namespace ocrsystem { - typedef pipeline::PPOCRv5 PPOCRSystemv5; -} // namespace ocrsystem -} // namespace application - } // namespace fastdeploy + diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v6.h b/fastdeploy/vision/ocr/ppocr/ppocr_v6.h index d4ca7f6e3ee..9a08d7f0b87 100644 --- a/fastdeploy/vision/ocr/ppocr/ppocr_v6.h +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v6.h @@ -22,6 +22,15 @@ namespace fastdeploy { */ namespace pipeline { /*! @brief PPOCRv6 is used to load PP-OCRv6 series models provided by PaddleOCR. + * + * Note: For PP-OCRv6, PaddleOCR officially recommends detector postprocessing parameters: + * - det_db_thresh = 0.2 + * - det_db_box_thresh = 0.45 + * - det_db_unclip_ratio = 1.4 + * - det_db_max_candidates = 3000 + * To avoid overwriting custom configurations, PPOCRv6 does not modify DBDetector + * parameters in its constructor. Call det_model->GetPostprocessor().SetDetDB*() + * before or after pipeline construction if needed. */ class FASTDEPLOY_DECL PPOCRv6 : public PPOCRv5 { public: @@ -34,17 +43,7 @@ class FASTDEPLOY_DECL PPOCRv6 : public PPOCRv5 { PPOCRv6(fastdeploy::vision::ocr::DBDetector* det_model, fastdeploy::vision::ocr::Classifier* cls_model, fastdeploy::vision::ocr::Recognizer* rec_model) - : PPOCRv5(det_model, cls_model, rec_model) { - auto preprocess_shape = recognizer_->GetPreprocessor().GetRecImageShape(); - preprocess_shape[1] = 48; - recognizer_->GetPreprocessor().SetRecImageShape(preprocess_shape); - if (detector_ != nullptr) { - detector_->GetPostprocessor().SetDetDBThresh(0.2); - detector_->GetPostprocessor().SetDetDBBoxThresh(0.45); - detector_->GetPostprocessor().SetDetDBUnclipRatio(1.4); - detector_->GetPostprocessor().SetDetDBMaxCandidates(3000); - } - } + : PPOCRv5(det_model, cls_model, rec_model) {} /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively. * * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv6_det_infer @@ -52,17 +51,7 @@ class FASTDEPLOY_DECL PPOCRv6 : public PPOCRv5 { */ PPOCRv6(fastdeploy::vision::ocr::DBDetector* det_model, fastdeploy::vision::ocr::Recognizer* rec_model) - : PPOCRv5(det_model, rec_model) { - auto preprocess_shape = recognizer_->GetPreprocessor().GetRecImageShape(); - preprocess_shape[1] = 48; - recognizer_->GetPreprocessor().SetRecImageShape(preprocess_shape); - if (detector_ != nullptr) { - detector_->GetPostprocessor().SetDetDBThresh(0.2); - detector_->GetPostprocessor().SetDetDBBoxThresh(0.45); - detector_->GetPostprocessor().SetDetDBUnclipRatio(1.4); - detector_->GetPostprocessor().SetDetDBMaxCandidates(3000); - } - } + : PPOCRv5(det_model, rec_model) {} /** \brief Clone a new PPOCRv6 with less memory usage when multiple instances of the same model are created * @@ -81,10 +70,5 @@ class FASTDEPLOY_DECL PPOCRv6 : public PPOCRv5 { } // namespace pipeline -namespace application { -namespace ocrsystem { - typedef pipeline::PPOCRv6 PPOCRSystemv6; -} // namespace ocrsystem -} // namespace application - } // namespace fastdeploy + From 20a911f2f46ec2c0cf4d8bdd247547db935dd82a Mon Sep 17 00:00:00 2001 From: SherkeyXD <57581480+SherkeyXD@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:59:40 +0800 Subject: [PATCH 2/5] feat(ppocr): add translation units for PPOCRv5 and PPOCRv6 to ensure compilation --- fastdeploy/vision/ocr/ppocr/ppocr_v5.cc | 40 +++++++++++++++++++++++++ fastdeploy/vision/ocr/ppocr/ppocr_v5.h | 16 ++-------- fastdeploy/vision/ocr/ppocr/ppocr_v6.cc | 40 +++++++++++++++++++++++++ fastdeploy/vision/ocr/ppocr/ppocr_v6.h | 16 ++-------- 4 files changed, 86 insertions(+), 26 deletions(-) create mode 100644 fastdeploy/vision/ocr/ppocr/ppocr_v5.cc create mode 100644 fastdeploy/vision/ocr/ppocr/ppocr_v6.cc diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v5.cc b/fastdeploy/vision/ocr/ppocr/ppocr_v5.cc new file mode 100644 index 00000000000..aa5aeb30f82 --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v5.cc @@ -0,0 +1,40 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/ocr/ppocr/ppocr_v5.h" + +namespace fastdeploy { +namespace pipeline { + +PPOCRv5::PPOCRv5(fastdeploy::vision::ocr::DBDetector* det_model, + fastdeploy::vision::ocr::Classifier* cls_model, + fastdeploy::vision::ocr::Recognizer* rec_model) + : PPOCRv4(det_model, cls_model, rec_model) {} + +PPOCRv5::PPOCRv5(fastdeploy::vision::ocr::DBDetector* det_model, + fastdeploy::vision::ocr::Recognizer* rec_model) + : PPOCRv4(det_model, rec_model) {} + +std::unique_ptr PPOCRv5::Clone() const { + std::unique_ptr clone_model = utils::make_unique(PPOCRv5(*this)); + clone_model->detector_ = detector_->Clone().release(); + if (classifier_ != nullptr) { + clone_model->classifier_ = classifier_->Clone().release(); + } + clone_model->recognizer_ = recognizer_->Clone().release(); + return clone_model; +} + +} // namespace pipeline +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v5.h b/fastdeploy/vision/ocr/ppocr/ppocr_v5.h index b0688787d06..51bc0e355c9 100644 --- a/fastdeploy/vision/ocr/ppocr/ppocr_v5.h +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v5.h @@ -33,30 +33,20 @@ class FASTDEPLOY_DECL PPOCRv5 : public PPOCRv4 { */ PPOCRv5(fastdeploy::vision::ocr::DBDetector* det_model, fastdeploy::vision::ocr::Classifier* cls_model, - fastdeploy::vision::ocr::Recognizer* rec_model) - : PPOCRv4(det_model, cls_model, rec_model) {} + fastdeploy::vision::ocr::Recognizer* rec_model); /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively. * * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv5_det_infer * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv5_rec_infer */ PPOCRv5(fastdeploy::vision::ocr::DBDetector* det_model, - fastdeploy::vision::ocr::Recognizer* rec_model) - : PPOCRv4(det_model, rec_model) {} + fastdeploy::vision::ocr::Recognizer* rec_model); /** \brief Clone a new PPOCRv5 with less memory usage when multiple instances of the same model are created * * \return new PPOCRv5* type unique pointer */ - std::unique_ptr Clone() const { - std::unique_ptr clone_model = utils::make_unique(PPOCRv5(*this)); - clone_model->detector_ = detector_->Clone().release(); - if (classifier_ != nullptr) { - clone_model->classifier_ = classifier_->Clone().release(); - } - clone_model->recognizer_ = recognizer_->Clone().release(); - return clone_model; - } + std::unique_ptr Clone() const; }; } // namespace pipeline diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v6.cc b/fastdeploy/vision/ocr/ppocr/ppocr_v6.cc new file mode 100644 index 00000000000..ab5571c9871 --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v6.cc @@ -0,0 +1,40 @@ +// Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/ocr/ppocr/ppocr_v6.h" + +namespace fastdeploy { +namespace pipeline { + +PPOCRv6::PPOCRv6(fastdeploy::vision::ocr::DBDetector* det_model, + fastdeploy::vision::ocr::Classifier* cls_model, + fastdeploy::vision::ocr::Recognizer* rec_model) + : PPOCRv5(det_model, cls_model, rec_model) {} + +PPOCRv6::PPOCRv6(fastdeploy::vision::ocr::DBDetector* det_model, + fastdeploy::vision::ocr::Recognizer* rec_model) + : PPOCRv5(det_model, rec_model) {} + +std::unique_ptr PPOCRv6::Clone() const { + std::unique_ptr clone_model = utils::make_unique(PPOCRv6(*this)); + clone_model->detector_ = detector_->Clone().release(); + if (classifier_ != nullptr) { + clone_model->classifier_ = classifier_->Clone().release(); + } + clone_model->recognizer_ = recognizer_->Clone().release(); + return clone_model; +} + +} // namespace pipeline +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v6.h b/fastdeploy/vision/ocr/ppocr/ppocr_v6.h index 9a08d7f0b87..41ea6842f9a 100644 --- a/fastdeploy/vision/ocr/ppocr/ppocr_v6.h +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v6.h @@ -42,30 +42,20 @@ class FASTDEPLOY_DECL PPOCRv6 : public PPOCRv5 { */ PPOCRv6(fastdeploy::vision::ocr::DBDetector* det_model, fastdeploy::vision::ocr::Classifier* cls_model, - fastdeploy::vision::ocr::Recognizer* rec_model) - : PPOCRv5(det_model, cls_model, rec_model) {} + fastdeploy::vision::ocr::Recognizer* rec_model); /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively. * * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv6_det_infer * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv6_rec_infer */ PPOCRv6(fastdeploy::vision::ocr::DBDetector* det_model, - fastdeploy::vision::ocr::Recognizer* rec_model) - : PPOCRv5(det_model, rec_model) {} + fastdeploy::vision::ocr::Recognizer* rec_model); /** \brief Clone a new PPOCRv6 with less memory usage when multiple instances of the same model are created * * \return new PPOCRv6* type unique pointer */ - std::unique_ptr Clone() const { - std::unique_ptr clone_model = utils::make_unique(PPOCRv6(*this)); - clone_model->detector_ = detector_->Clone().release(); - if (classifier_ != nullptr) { - clone_model->classifier_ = classifier_->Clone().release(); - } - clone_model->recognizer_ = recognizer_->Clone().release(); - return clone_model; - } + std::unique_ptr Clone() const; }; } // namespace pipeline From ded1968d118a1a7cc12f8692315a251fa81217b0 Mon Sep 17 00:00:00 2001 From: SherkeyXD <57581480+SherkeyXD@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:59:53 +0800 Subject: [PATCH 3/5] chore: remove unused llm module --- services/llm/.dockerignore | 11 - services/llm/README.md | 39 - services/llm/client/README.md | 110 --- .../llm/client/fastdeploy_client/__init__.py | 20 - .../llm/client/fastdeploy_client/chatbot.py | 304 -------- .../llm/client/fastdeploy_client/command.py | 71 -- .../llm/client/fastdeploy_client/message.py | 50 -- .../llm/client/fastdeploy_client/utils.py | 19 - services/llm/client/requirements.txt | 5 - services/llm/client/setup.py | 39 - .../Dockerfile_serving_cuda118_cudnn8 | 34 - .../Dockerfile_serving_cuda123_cudnn9 | 34 - .../llm/docs/FastDeploy_usage_tutorial.md | 357 --------- services/llm/requirements-dev.txt | 3 - services/llm/server/config/config.pbtxt | 20 - services/llm/server/requirements.txt | 21 - services/llm/server/scripts/start_server.sh | 57 -- services/llm/server/scripts/stop_server.sh | 68 -- services/llm/server/server/__init__.py | 16 - services/llm/server/server/checker.py | 124 ---- services/llm/server/server/data/__init__.py | 13 - services/llm/server/server/data/processor.py | 389 ---------- services/llm/server/server/engine/__init__.py | 13 - services/llm/server/server/engine/config.py | 265 ------- services/llm/server/server/engine/engine.py | 401 ---------- services/llm/server/server/engine/infer.py | 697 ------------------ .../server/server/engine/resource_manager.py | 241 ------ .../server/engine/task_queue_manager.py | 165 ----- .../server/server/engine/token_processor.py | 272 ------- .../llm/server/server/http_server/__init__.py | 13 - .../server/http_server/adapter_openai.py | 103 --- services/llm/server/server/http_server/api.py | 188 ----- services/llm/server/server/http_server/app.py | 131 ---- services/llm/server/server/triton_server.py | 490 ------------ .../llm/server/server/triton_server_helper.py | 157 ---- services/llm/server/server/utils.py | 195 ----- 36 files changed, 5135 deletions(-) delete mode 100644 services/llm/.dockerignore delete mode 100644 services/llm/README.md delete mode 100644 services/llm/client/README.md delete mode 100644 services/llm/client/fastdeploy_client/__init__.py delete mode 100644 services/llm/client/fastdeploy_client/chatbot.py delete mode 100644 services/llm/client/fastdeploy_client/command.py delete mode 100644 services/llm/client/fastdeploy_client/message.py delete mode 100644 services/llm/client/fastdeploy_client/utils.py delete mode 100644 services/llm/client/requirements.txt delete mode 100644 services/llm/client/setup.py delete mode 100644 services/llm/dockerfiles/Dockerfile_serving_cuda118_cudnn8 delete mode 100644 services/llm/dockerfiles/Dockerfile_serving_cuda123_cudnn9 delete mode 100644 services/llm/docs/FastDeploy_usage_tutorial.md delete mode 100644 services/llm/requirements-dev.txt delete mode 100644 services/llm/server/config/config.pbtxt delete mode 100644 services/llm/server/requirements.txt delete mode 100644 services/llm/server/scripts/start_server.sh delete mode 100644 services/llm/server/scripts/stop_server.sh delete mode 100644 services/llm/server/server/__init__.py delete mode 100644 services/llm/server/server/checker.py delete mode 100644 services/llm/server/server/data/__init__.py delete mode 100644 services/llm/server/server/data/processor.py delete mode 100644 services/llm/server/server/engine/__init__.py delete mode 100644 services/llm/server/server/engine/config.py delete mode 100644 services/llm/server/server/engine/engine.py delete mode 100644 services/llm/server/server/engine/infer.py delete mode 100644 services/llm/server/server/engine/resource_manager.py delete mode 100644 services/llm/server/server/engine/task_queue_manager.py delete mode 100644 services/llm/server/server/engine/token_processor.py delete mode 100644 services/llm/server/server/http_server/__init__.py delete mode 100644 services/llm/server/server/http_server/adapter_openai.py delete mode 100644 services/llm/server/server/http_server/api.py delete mode 100644 services/llm/server/server/http_server/app.py delete mode 100644 services/llm/server/server/triton_server.py delete mode 100644 services/llm/server/server/triton_server_helper.py delete mode 100644 services/llm/server/server/utils.py diff --git a/services/llm/.dockerignore b/services/llm/.dockerignore deleted file mode 100644 index 96dbf3cb734..00000000000 --- a/services/llm/.dockerignore +++ /dev/null @@ -1,11 +0,0 @@ -README.md -requirements-dev.txt -pyproject.toml -Makefile - -dockerfiles/ -docs/ -server/__pycache__ -server/http_server -server/engine -server/data diff --git a/services/llm/README.md b/services/llm/README.md deleted file mode 100644 index 6ed630ee7c2..00000000000 --- a/services/llm/README.md +++ /dev/null @@ -1,39 +0,0 @@ - -

FastDeploy大模型服务化部署

- -*FastDeploy基于英伟达Triton框架专为服务器场景的大模型服务化部署而设计的解决方案。它提供了支持gRPC、HTTP协议的服务接口，以及流式Token输出能力。底层推理引擎支持连续批处理、weight only int8、后训练量化（PTQ）等加速优化策略，为用户带来易用且高性能的部署体验。* - -# 快速开始 - - 基于预编译镜像部署，本节以 Meta-Llama-3-8B-Instruct-A8W8C8 为例，更多模型请参考[LLaMA](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/docs/predict/llama.md)、[Qwen](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/docs/predict/qwen.md)、[Mixtral](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/docs/predict/mixtral.md), 更细致的模型推理、量化教程可以参考[大模型推理教程](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/docs/predict/inference.md)： - - ``` - # 下载模型 - wget https://paddle-qa.bj.bcebos.com/inference_model/Meta-Llama-3-8B-Instruct-A8W8C8.tar - mkdir Llama-3-8B-A8W8C8 && tar -xf Meta-Llama-3-8B-Instruct-A8W8C8.tar -C Llama-3-8B-A8W8C8 - - # 挂载模型文件 - export MODEL_PATH=${PWD}/Llama-3-8B-A8W8C8 - - docker run --gpus all --shm-size 5G --network=host --privileged --cap-add=SYS_PTRACE \ - -v ${MODEL_PATH}:/models/ \ - -dit registry.baidubce.com/paddlepaddle/fastdeploy:llm-serving-cuda123-cudnn9-v1.2 \ - bash -c 'export USE_CACHE_KV_INT8=1 && cd /opt/output/Serving && bash start_server.sh; exec bash' - ``` - - 等待服务启动成功（服务初次启动大概需要40s），可以通过以下命令测试： - - ``` - curl 127.0.0.1:9965/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -d '{"text": "hello, llm"}' - ``` - -Note: -1. 请保证 shm-size >= 5，不然可能会导致服务启动失败 - -更多关于 FastDeploy 的使用方法，请查看[服务化部署流程](https://github.com/PaddlePaddle/FastDeploy/blob/develop/llm/docs/FastDeploy_usage_tutorial.md) - -# License - -FastDeploy 遵循 [Apache-2.0开源协议](https://github.com/PaddlePaddle/FastDeploy/blob/develop/LICENSE) 。 diff --git a/services/llm/client/README.md b/services/llm/client/README.md deleted file mode 100644 index 396f83cc091..00000000000 --- a/services/llm/client/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# 客户端使用方式 - -## 简介 - -FastDeploy客户端提供命令行接口和Python接口，可以快速调用FastDeploy后端部署的LLM模型服务。 - -## 安装 - -源码安装 -``` -pip install . -``` - -## 命令行接口 - -首先通过环境变量设置模型服务模式、模型服务URL、模型ID，然后使用命令行接口调用模型服务。 - -| 参数 | 说明 | 是否必填 | 默认值 | -| --- | --- | --- | --- | -| FASTDEPLOY_MODEL_URL | 模型服务部署的IP地址和端口，格式为`x.x.x.x:xxx`。 | 是 | | - -``` -export FASTDEPLOY_MODEL_URL="x.x.x.x:xxx" - -# 流式接口 -fdclient stream_generate "你好?" - -# 非流式接口 -fdclient generate "你好，你是谁?" -``` - -## Python接口 - -首先通过Python代码设置模型服务URL(hostname+port)，然后使用Python接口调用模型服务。 - -| 参数 | 说明 | 是否必填 | 默认值 | -| --- | --- | --- | --- | -| hostname+port | 模型服务部署的IP地址和端口，格式为`x.x.x.x。 | 是 | | - - -``` -from fastdeploy_client.chatbot import ChatBot - -hostname = "x.x.x.x" -port = xxx - -# 流式接口，stream_generate api的参数说明见附录 -chatbot = ChatBot(hostname=hostname, port=port) -stream_result = chatbot.stream_generate("你好", topp=0.8) -for res in stream_result: - print(res) - -# 非流式接口，generate api的参数说明见附录 -chatbot = ChatBot(hostname=hostname, port=port) -result = chatbot.generate("你好", topp=0.8) -print(result) -``` - -### 接口说明 -``` -ChatBot.stream_generate(message, - max_dec_len=1024, - min_dec_len=2, - topp=0.0, - temperature=1.0, - frequency_score=0.0, - penalty_score=1.0, - presence_score=0.0, - eos_token_ids=254186) - -# 此函数返回一个iterator，其中每个元素为一个dict, 例如：{"token": "好的", "is_end": 0} -# 其中token为生成的字符，is_end表明是否为生成的最后一个字符（0表示否，1表示是） -# 注意：当生成结果出错时，返回错误信息；不同模型的eos_token_ids不同 -``` - -``` -ChatBot.generate(message, - max_dec_len=1024, - min_dec_len=2, - topp=0.0, - temperature=1.0, - frequency_score=0.0, - penalty_score=1.0, - presence_score=0.0, - eos_token_ids=254186) - -# 此函数返回一个，例如：{"results": "好的，我知道了。"}，其中results即为生成结果 -# 注意：当生成结果出错时，返回错误信息；不同模型的eos_token_ids不同 -``` - -### 参数说明 - -| 字段名 | 字段类型 | 说明 | 是否必填 | 默认值 | 备注 | -| :---: | :-----: | :---: | :---: | :-----: | :----: | -| req_id | str | 请求ID，用于标识一个请求。建议设置req_id，保证其唯一性 | 否 | 随机id | 如果推理服务中同时有两个相同req_id的请求，会返回req_id重复的错误信息 | -| text | str | 请求的文本 | 是 | 无 | | -| max_dec_len | int | 最大生成token的长度，如果请求的文本token长度加上max_dec_len大于模型的max_seq_len，会返回长度超限的错误信息 | 否 | max_seq_len减去文本token长度 | | -| min_dec_len | int | 最小生成token的长度，最小是1 | 否 | 1 | | -| topp | float | 控制随机性参数，数值越大则随机性越大，范围是0~1 | 否 | 0.7 | | -| temperature | float | 控制随机性参数，数值越小随机性越大，需要大于 0 | 否 | 0.95 | | -| frequency_score | float | 频率分数 | 否 | 0 | | -| penalty_score | float | 惩罚分数 | 否 | 1 | | -| presence_score | float | 存在分数 | 否 | 0 | | -| stream | bool | 是否流式返回 | 否 | False | | -| return_all_tokens | bool | 是否一次性返回所有结果 | 否 | False | 与stream参数差异见表后备注 | -| timeout | int | 请求等待的超时时间，单位是秒 | 否 | 300 | | - -* 在正确配置PUSH_MODE_HTTP_PORT字段下，服务支持 GRPC 和 HTTP 两种请求服务 - * stream 参数仅对 HTTP 请求生效 - * return_all_tokens 参数对 GRPC 和 HTTP 请求均有效 diff --git a/services/llm/client/fastdeploy_client/__init__.py b/services/llm/client/fastdeploy_client/__init__.py deleted file mode 100644 index 83ae7a0036f..00000000000 --- a/services/llm/client/fastdeploy_client/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import sys - -__version__ = "4.4.0" - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) diff --git a/services/llm/client/fastdeploy_client/chatbot.py b/services/llm/client/fastdeploy_client/chatbot.py deleted file mode 100644 index 0223e6b0f62..00000000000 --- a/services/llm/client/fastdeploy_client/chatbot.py +++ /dev/null @@ -1,304 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging -import queue -import traceback -import uuid -from functools import partial - -import numpy as np -import tritonclient.grpc as grpcclient -from fastdeploy_client.message import ChatMessage -from fastdeploy_client.utils import is_enable_benchmark -from tritonclient import utils as triton_utils - - -class ChatBotClass(object): - """ - initiating conversations through the tritonclient interface of the model service. - """ - def __init__(self, hostname, port, timeout=120): - """ - Initialization function - - Args: - hostname (str): gRPC hostname - port (int): gRPC port - timeout (int): Request timeout, default is 120 seconds. - - Returns: - None - """ - self.url = f"{hostname}:{port}" - self.timeout = timeout - - def stream_generate(self, - message, - max_dec_len=1024, - min_dec_len=1, - topp=0.7, - temperature=0.95, - frequency_score=0.0, - penalty_score=1.0, - presence_score=0.0, - system=None, - **kwargs): - """ - Streaming interface - - Args: - message (Union[str, List[str], ChatMessage]): message or ChatMessage object - max_dec_len (int, optional): max decoding length. Defaults to 1024. - min_dec_len (int, optional): min decoding length. Defaults to 1. - topp (float, optional): randomness of the generated tokens. Defaults to 0.7. - temperature (float, optional): temperature. Defaults to 0.95. - frequency_score (float, optional): frequency score. Defaults to 0.0. - penalty_score (float, optional): penalty score. Defaults to 1.0. - presence_score (float, optional): presence score. Defaults to 0.0. - system (str, optional): system settings. Defaults to None. - **kwargs: others - - For more details, please refer to https://github.com/PaddlePaddle/FastDeploy/blob/develop/llm/docs/FastDeploy_usage_tutorial.md#%E8%AF%B7%E6%B1%82%E5%8F%82%E6%95%B0%E4%BB%8B%E7%BB%8D - - Returns: - return a generator object, which yields a dict. - Normal, return {'token': xxx, 'is_end': xxx, 'send_idx': xxx, ..., 'error_msg': '', 'error_code': 0} - Others, return {'error_msg': xxx, 'error_code': xxx}, error_msg not None, error_code != 0 - """ - try: - model_name = "model" - inputs = [grpcclient.InferInput("IN", [1], triton_utils.np_to_triton_dtype(np.object_))] - outputs = [grpcclient.InferRequestedOutput("OUT")] - output_data = OutputData() - - msg = message.message if isinstance(message, ChatMessage) else message - input_data = self._prepare_input_data(msg, max_dec_len, min_dec_len, - topp, temperature, frequency_score, - penalty_score, presence_score, **kwargs) - req_id = input_data["req_id"] - inputs[0].set_data_from_numpy(np.array([json.dumps([input_data])], dtype=np.object_)) - timeout = kwargs.get("timeout", self.timeout) - - with grpcclient.InferenceServerClient(url=self.url, verbose=False) as triton_client: - triton_client.start_stream(callback=partial(triton_callback, output_data)) - triton_client.async_stream_infer(model_name=model_name, - inputs=inputs, - request_id=req_id, - outputs=outputs) - answer_str = "" - enable_benchmark = is_enable_benchmark(**kwargs) - while True: - try: - response = output_data._completed_requests.get(timeout=timeout) - except queue.Empty: - yield {"req_id": req_id, "error_msg": f"Fetch response from server timeout ({timeout}s)"} - break - if type(response) == triton_utils.InferenceServerException: - yield {"req_id": req_id, "error_msg": f"InferenceServerException raised by inference: {response.message()}"} - break - else: - if enable_benchmark: - response = json.loads(response.as_numpy("OUT")[0]) - if isinstance(response, (list, tuple)): - response = response[0] - else: - response = self._format_response(response, req_id) - token = response.get("token", "") - if isinstance(token, list): - token = token[0] - answer_str += token - yield response - if response.get("is_end") == 1 or response.get("error_msg") is not None: - break - triton_client.stop_stream(cancel_requests=True) - triton_client.close() - - if isinstance(message, ChatMessage): - message.message.append({"role": "assistant", "content": answer_str}) - except Exception as e: - yield {"error_msg": f"{e}, details={str(traceback.format_exc())}"} - - def generate(self, - message, - max_dec_len=1024, - min_dec_len=1, - topp=0.7, - temperature=0.95, - frequency_score=0.0, - penalty_score=1.0, - presence_score=0.0, - system=None, - **kwargs): - """ - Return the entire sentence using the streaming interface. - - Args: - message (Union[str, List[str], ChatMessage]): message or ChatMessage object - max_dec_len (int, optional): max decoding length. Defaults to 1024. - min_dec_len (int, optional): min decoding length. Defaults to 1. - topp (float, optional): randomness of the generated tokens. Defaults to 0.7. - temperature (float, optional): temperature. Defaults to 0.95. - frequency_score (float, optional): frequency score. Defaults to 0.0. - penalty_score (float, optional): penalty score. Defaults to 1.0. - presence_score (float, optional): presence score. Defaults to 0.0. - system (str, optional): system settings. Defaults to None. - **kwargs: others - - For more details, please refer to https://github.com/PaddlePaddle/FastDeploy/blob/develop/llm/docs/FastDeploy_usage_tutorial.md#%E8%AF%B7%E6%B1%82%E5%8F%82%E6%95%B0%E4%BB%8B%E7%BB%8D - - Returns: - return the entire sentence or error message. - Normal, return {'tokens_all': xxx, ..., 'error_msg': '', 'error_code': 0} - Others, return {'error_msg': xxx, 'error_code': xxx}, error_msg not None, error_code != 0 - """ - stream_response = self.stream_generate(message, max_dec_len, - min_dec_len, topp, temperature, - frequency_score, penalty_score, - presence_score, system, **kwargs) - results = "" - token_ids = list() - error_msg = None - for res in stream_response: - if "token" not in res or "error_msg" in res: - error_msg = {"error_msg": f"response error, please check the info: {res}"} - elif isinstance(res["token"], list): - results = res["token"] - token_ids = res["token_ids"] - else: - results += res["token"] - token_ids += res["token_ids"] - if error_msg: - return {"req_id": res["req_id"], "error_msg": error_msg} - else: - return {"req_id": res["req_id"], "results": results, "token_ids": token_ids} - - def _prepare_input_data(self, - message, - max_dec_len=1024, - min_dec_len=2, - topp=0.0, - temperature=1.0, - frequency_score=0.0, - penalty_score=1.0, - presence_score=0.0, - system=None, - **kwargs): - """ - Prepare to input data - """ - inputs = { - "max_dec_len": max_dec_len, - "min_dec_len": min_dec_len, - "topp": topp, - "temperature": temperature, - "frequency_score": frequency_score, - "penalty_score": penalty_score, - "presence_score": presence_score, - } - - if system is not None: - inputs["system"] = system - - inputs["req_id"] = kwargs.get("req_id", str(uuid.uuid4())) - if "eos_token_ids" in kwargs and kwargs["eos_token_ids"] is not None: - inputs["eos_token_ids"] = kwargs["eos_token_ids"] - inputs["response_timeout"] = kwargs.get("timeout", self.timeout) - - if isinstance(message, str): - inputs["text"] = message - elif isinstance(message, list): - assert len(message) % 2 == 1, \ - "The length of message should be odd while it's a list." - assert message[-1]["role"] == "user", \ - "The {}-th element key should be 'user'".format(len(message) - 1) - for i in range(0, len(message) - 1, 2): - assert message[i]["role"] == "user", \ - "The {}-th element key should be 'user'".format(i) - assert message[i + 1]["role"] == "assistant", \ - "The {}-th element key should be 'assistant'".format(i + 1) - inputs["messages"] = message - else: - raise Exception( - "The message should be string or list of dict like [{'role': " - "'user', 'content': 'Hello, what's your name?''}]" - ) - - return inputs - - def _format_response(self, response, req_id): - """ - Format the service return fields - """ - response = json.loads(response.as_numpy("OUT")[0]) - if isinstance(response, (list, tuple)): - response = response[0] - is_end = response.get("is_end", False) - - if "error_msg" in response: - return {"req_id": req_id, "error_msg": response["error_msg"]} - elif "choices" in response: - token = [x["token"] for x in response["choices"]] - token_ids = [x["token_ids"] for x in response["choices"]] - return {"req_id": req_id, "token": token, "token_ids": token_ids, "is_end": 1} - elif "token" not in response and "result" not in response: - return {"req_id": req_id, "error_msg": f"The response should contain 'token' or 'result', but got {response}"} - else: - token_ids = response.get("token_ids", []) - if "result" in response: - token = response["result"] - elif "token" in response: - token = response["token"] - return {"req_id": req_id, "token": token, "token_ids": token_ids, "is_end": is_end} - - -class OutputData: - """ - Receive data returned by Triton service - """ - def __init__(self): - self._completed_requests = queue.Queue() - - -def triton_callback(output_data, result, error): - """ - callback function for Triton server - """ - if error: - output_data._completed_requests.put(error) - else: - output_data._completed_requests.put(result) - - -class ChatBot(object): - """ - External interface, create a client object ChatBotForPushMode - """ - def __new__(cls, hostname, port, timeout=120): - """ - initialize a GRPCInferenceService client - Args: - hostname (str): server hostname - port (int): GRPC port - timeout (int): timeout(s), default 120 seconds - Returns: - ChatBotClass: BaseChatBot object - """ - if not isinstance(hostname, str) or not hostname: - raise ValueError("Invalid hostname") - if not isinstance(port, int) or port <= 0 or port > 65535: - raise ValueError("Invalid port number") - - return ChatBotClass(hostname, port, timeout) diff --git a/services/llm/client/fastdeploy_client/command.py b/services/llm/client/fastdeploy_client/command.py deleted file mode 100644 index 8aaf9ab7c2a..00000000000 --- a/services/llm/client/fastdeploy_client/command.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import sys - -from fastdeploy_client.chatbot import ChatBot - - -def _get_service_configuration(): - """ - get service url from environment - - Returns: - tuple: (hostname, port) - """ - url = os.getenv("FASTDEPLOY_MODEL_URL") - - if url is None: - raise ValueError("Please set service url by `export FASTDEPLOY_MODEL_URL`." - "For example: `export FASTDEPLOY_MODEL_URL=127.0.0.1:8500`") - hostname, port = url.strip().split(':') - port = int(port) - if port <= 0 or port > 65535: - raise ValueError("Invalid port number") - - return hostname, port - - -def stream_generate(prompt): - """ - Streaming interface - """ - hostname, port = _get_service_configuration() - chatbot = ChatBot(hostname=hostname, port=port) - stream_result = chatbot.stream_generate(prompt) - for res in stream_result: - print(res) - - -def generate(prompt): - """ - entire sentence interface - """ - hostname, port = _get_service_configuration() - chatbot = ChatBot(hostname=hostname, port=port) - result = chatbot.generate(prompt) - print(result) - - -def main(): - if len(sys.argv) < 2 or sys.argv[1] not in ["generate", "stream_generate"]: - logging.error("Usage 1: fdclient generate \"Hello, How are you?\"") - return - prompt = sys.argv[2] - if sys.argv[1] == "generate": - return generate(prompt) - else: - return stream_generate(prompt) diff --git a/services/llm/client/fastdeploy_client/message.py b/services/llm/client/fastdeploy_client/message.py deleted file mode 100644 index 7ce1b7b7b91..00000000000 --- a/services/llm/client/fastdeploy_client/message.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -class ChatMessage(object): - """ - multi-turn chat message with ChatBot - """ - def __init__(self, prompt=None): - if prompt is not None: - self.message = [{"role": "user", "content": prompt}] - else: - self.message = [] - - def add_user_message(self, content): - """ - add user message - """ - if len(self.message) > 0 and self.message[-1]["role"] != "assistant": - raise Exception("Cannot add user message, because the role of the " - f"last message is not assistant. The message is {self.message}") - self.message.append({"role": "user", "content": content}) - - def add_assistant_message(self, content): - """ - add assistant message - """ - if len(self.message) > 0 and self.message[-1]["role"] != "user": - raise Exception("Cannot add user message, because the role of the " - f"last message is not user. The message is {self.message}") - self.message.append({"role": "assistant", "content": content}) - - def next_prompt(self, content): - """ - add user message and return a new prompt - """ - self.add_user_message(content) - - def __str__(self): - return str(self.message) diff --git a/services/llm/client/fastdeploy_client/utils.py b/services/llm/client/fastdeploy_client/utils.py deleted file mode 100644 index b5c3f33165f..00000000000 --- a/services/llm/client/fastdeploy_client/utils.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -def is_enable_benchmark(**kwargs): - """ - Check if enable benchmark - """ - return "benchmark" in kwargs and kwargs["benchmark"] == 1 diff --git a/services/llm/client/requirements.txt b/services/llm/client/requirements.txt deleted file mode 100644 index 132f7f2b0da..00000000000 --- a/services/llm/client/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -grpcio -streamlit<=1.33.0 -streamlit_chat<=0.1.1 -protobuf==3.20.0 -tritonclient[grpc]==2.41.1 diff --git a/services/llm/client/setup.py b/services/llm/client/setup.py deleted file mode 100644 index 9075c45ae05..00000000000 --- a/services/llm/client/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import setuptools -from fastdeploy_client import __version__ as version - -long_description = "No description" -with open("requirements.txt") as fin: - REQUIRED_PACKAGES = fin.read() - -setuptools.setup( - name="fastdeploy-client", - version=version, - author="dltp-sz", - author_email="dltp-sz@baidu.com", - description="Client for fastdeploy llm serving", - long_description=long_description, - long_description_content_type="text/plain", - url="https://github.com/PaddlePaddle/Paddle", - packages=setuptools.find_packages(), - install_requires=REQUIRED_PACKAGES, - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - ], - license='Apache 2.0', - entry_points={'console_scripts': ['fdclient=fastdeploy_client.command:main', ]}) diff --git a/services/llm/dockerfiles/Dockerfile_serving_cuda118_cudnn8 b/services/llm/dockerfiles/Dockerfile_serving_cuda118_cudnn8 deleted file mode 100644 index b7cd4205c4d..00000000000 --- a/services/llm/dockerfiles/Dockerfile_serving_cuda118_cudnn8 +++ /dev/null @@ -1,34 +0,0 @@ -FROM registry.baidubce.com/paddlepaddle/fastdeploy:llm-base-gcc12.3-cuda11.8-cudnn8-nccl2.15.5 - -WORKDIR /opt/output/ -COPY ./server/ /opt/output/Serving/ -COPY ./client/ /opt/output/client/ - -ENV LD_LIBRARY_PATH="/usr/local/cuda-11.8/compat/:$LD_LIBRARY_PATH" - -RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -RUN python3 -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu118/ \ - && python3 -m pip install paddlenlp==3.0.0b0 \ - && python3 -m pip install --no-cache-dir sentencepiece pycryptodome tritonclient[all]==2.41.1 - -RUN git clone https://gitee.com/paddlepaddle/PaddleNLP.git && cd PaddleNLP/csrc \ - && python3 setup_cuda.py build && python3 setup_cuda.py install --user \ - && cp -r /opt/output/PaddleNLP/paddlenlp /usr/local/lib/python3.10/dist-packages/ \ - && cp -r /root/.local/lib/python3.10/site-packages/* /usr/local/lib/python3.10/dist-packages/ \ - && rm -rf /opt/output/PaddleNLP - -RUN cd /opt/output/client && pip install -r requirements.txt && pip install . - -RUN python3 -m pip install -r /opt/output/Serving/requirements.txt && rm /opt/output/Serving/requirements.txt -RUN mv Serving/server /usr/local/lib/python3.10/dist-packages/ -RUN mkdir -p /opt/output/Serving/llm_model/model/1 \ - && mv /opt/output/Serving/config/config.pbtxt /opt/output/Serving/llm_model/model/ \ - && rm -rf /opt/output/Serving/config/ -RUN echo "from server.triton_server import TritonPythonModel" >>/opt/output/Serving/llm_model/model/1/model.py - -RUN cd /opt/output/Serving/ \ - && cp scripts/start_server.sh . && cp scripts/stop_server.sh . \ - && rm -rf scripts - -ENV http_proxy="" -ENV https_proxy="" diff --git a/services/llm/dockerfiles/Dockerfile_serving_cuda123_cudnn9 b/services/llm/dockerfiles/Dockerfile_serving_cuda123_cudnn9 deleted file mode 100644 index fabb7c1724f..00000000000 --- a/services/llm/dockerfiles/Dockerfile_serving_cuda123_cudnn9 +++ /dev/null @@ -1,34 +0,0 @@ -FROM registry.baidubce.com/paddlepaddle/fastdeploy:llm-base-gcc12.3-cuda12.3-cudnn9-nccl2.15.5 - -WORKDIR /opt/output/ -COPY ./server/ /opt/output/Serving/ -COPY ./client/ /opt/output/client/ - -ENV LD_LIBRARY_PATH="/usr/local/cuda-12.3/compat/:$LD_LIBRARY_PATH" - -RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -RUN python3 -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu123/ \ - && python3 -m pip install paddlenlp==3.0.0b0 \ - && python3 -m pip install --no-cache-dir sentencepiece pycryptodome tritonclient[all]==2.41.1 - -RUN git clone https://gitee.com/paddlepaddle/PaddleNLP.git && cd PaddleNLP/csrc \ - && python3 setup_cuda.py build && python3 setup_cuda.py install --user \ - && cp -r /opt/output/PaddleNLP/paddlenlp /usr/local/lib/python3.10/dist-packages/ \ - && cp -r /root/.local/lib/python3.10/site-packages/* /usr/local/lib/python3.10/dist-packages/ \ - && rm -rf /opt/output/PaddleNLP - -RUN cd /opt/output/client && pip install -r requirements.txt && pip install . - -RUN python3 -m pip install -r /opt/output/Serving/requirements.txt && rm /opt/output/Serving/requirements.txt -RUN mv Serving/server /usr/local/lib/python3.10/dist-packages/ -RUN mkdir -p /opt/output/Serving/llm_model/model/1 \ - && mv /opt/output/Serving/config/config.pbtxt /opt/output/Serving/llm_model/model/ \ - && rm -rf /opt/output/Serving/config/ -RUN echo "from server.triton_server import TritonPythonModel" >>/opt/output/Serving/llm_model/model/1/model.py - -RUN cd /opt/output/Serving/ \ - && cp scripts/start_server.sh . && cp scripts/stop_server.sh . \ - && rm -rf scripts - -ENV http_proxy="" -ENV https_proxy="" diff --git a/services/llm/docs/FastDeploy_usage_tutorial.md b/services/llm/docs/FastDeploy_usage_tutorial.md deleted file mode 100644 index ec9fe5ac327..00000000000 --- a/services/llm/docs/FastDeploy_usage_tutorial.md +++ /dev/null @@ -1,357 +0,0 @@ - -## 目录 - -- [部署环境准备](#部署环境准备) - - [基础环境](#基础环境) - - [准备部署镜像](#准备部署镜像) - - [准备模型](#准备模型) - - [创建容器](#创建容器) -- [启动服务](#启动服务) - - [配置参数](#配置参数) - - [启动服务](#启动服务) - - [服务状态查询](#服务状态查询) -- [服务测试](#服务测试) - - [Python 客户端](#Python-客户端) - - [HTTP调用](#HTTP调用) - - [OpenAI 客户端](#OpenAI-客户端) - - [返回示例](#返回示例) -- [基于dockerfile创建自己的镜像](#基于dockerfile创建自己的镜像) -- [模型配置参数介绍](#模型配置参数介绍) -- [请求参数介绍](#请求参数介绍) - -## 部署环境准备 - -### 基础环境 - 目前 FastDeploy 仅支持在 Linux 系统下部署，部署之前请确保系统有正确的 GPU 环境。 - - - 安装 docker - 请参考 [Install Docker Engine](https://docs.docker.com/engine/install/) 选择对应的 Linux 平台安装 docker 环境。 - - - 安装 NVIDIA Container Toolkit - 请参考 [Installing the NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#installing-the-nvidia-container-toolkit) 了解并安装 NVIDIA Container Toolkit。 - - NVIDIA Container Toolkit 安装成功后，参考 [Running a Sample Workload with Docker](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/sample-workload.html#running-a-sample-workload-with-docker) 测试 NVIDIA Container Toolkit 是否可以正常使用。 - -### 准备部署镜像 - -为了方便部署，我们提供了 cuda12.3 的镜像，可以直接拉取镜像，或者使用我们提供的 `Dockerfile` [构建自定义镜像](#基于dockerfile创建自己的镜像) -``` -docker pull registry.baidubce.com/paddlepaddle/fastdeploy:llm-serving-cuda123-cudnn9-v1.2 -``` - -### 准备模型 - -FastDeploy 为 PaddleNLP 静态图模型提供了高效的部署方案，模型静态图导出方案请参考：[LLaMA](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/docs/predict/llama.md)、[Qwen](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/docs/predict/qwen.md)、[Mixtral](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/llm/docs/predict/mixtral.md) ... - -导出后的模型放在任意文件夹下，以 `/home/workspace/models_dir` 为例 - -``` -cd /home/workspace/models_dir - -# 导出的模型目录结构如下所示，理论上无缝支持 PaddleNLP 导出的静态图模型，无需修改模型目录结构 -# /opt/output/Serving/models -# ├── config.json # 模型配置文件 -# ├── xxxx.model # 词表模型文件 -# ├── special_tokens_map.json # 词表配置文件 -# ├── tokenizer_config.json # 词表配置文件 -# ├── rank_mapping.csv # 多卡模型会有此文件，如为单卡模型，则无此文件（可选，仅在多卡部署模式下需要） -# └── rank_0 # 保存模型结构和权重文件的目录 -# ├── model.pdiparams -# └── model.pdmodel -``` - -### 创建容器 - -创建容器之前，请检查 Docker 版本和 GPU 环境，确保 Docker 支持 `--gpus all` 参数。 - -将模型目录挂载到容器中，默认模型挂载地址为 `/models/`，服务启动时可通过 `MODEL_DIR` 环境变量自定义挂载地址。 -``` -docker run --gpus all \ - --name fastdeploy_serving \ - --privileged \ - --cap-add=SYS_PTRACE \ - --network=host \ - --shm-size=5G \ - -v /home/workspace/models_dir:/models/ \ - -dit registry.baidubce.com/paddlepaddle/fastdeploy:llm-serving-cuda123-cudnn9-v1.2 bash - -# 进入容器，检查GPU环境和模型挂载是否正常 -docker exec -it fastdeploy_serving /bin/bash -nvidia-smi -ls /models/ -``` - -## 启动服务 - -### 配置参数 - -根据需求和硬件信息，配置以下环境变量 - -``` -# 单/多卡推理配置。自行修改。 -## 如果是单卡推理，使用0卡，设置如下环境变量。 -export MP_NUM=1 -export CUDA_VISIBLE_DEVICES=0 - -## 如果是多卡推理，除了模型导出得满足2卡要求，同时设置如下环境变量。 -# export MP_NUM=2 -# export CUDA_VISIBLE_DEVICES=0,1 - -# 如部署场景无流式Token返回需求，可配置如下开关 -# 服务将会将每个请求的所有生成Token一次性返回 -# 降低服务逐个Token发送压力 -# 默认关闭 -# export DISABLE_STREAMING=1 - -# 配置数据服务。需要自行修改HTTP_PORT、GRPC_PORT、METRICS_PORT和INFER_QUEUE_PORT。(请事先检查端口可用) -export HTTP_PORT="8110" # 探活服务的http端口（当前仅用于健康检查、探活） -export GRPC_PORT="8811" # 模型推服务的grpc端口 -export METRICS_PORT="8722" # 模型服务中监督指标的端口 -export INFER_QUEUE_PORT="8813" # 模型服务内部使用的端口 -export PUSH_MODE_HTTP_PORT="9965" # 服务请求HTTP端口号，如不配置，默认为-1，即服务只支持GRPC协议 - -# MAX_SEQ_LEN: 服务会拒绝input token数量超过MAX_SEQ_LEN的请求，并返回错误提示 -# MAX_DEC_LEN: 服务会拒绝请求中max_dec_len/min_dec_len超过此参数的请求，并返回错误提示 -export MAX_SEQ_LEN=8192 -export MAX_DEC_LEN=1024 - -export BATCH_SIZE="48" # 设置最大Batch Size，模型可同时并发处理的最大输入数量，不能高于128 -export BLOCK_BS="5" # 缓存Block支持的最大Query Batch Size，如果出现out of memeory 错误，尝试减少该数值 -export BLOCK_RATIO="0.75" # 一般可以设置成 输入平均Token数/（输入+输出平均Token数) - -export MAX_CACHED_TASK_NUM="128" # 服务缓存队列最大长度，队列达到上限后，会拒绝新的请求，默认128 -# 开启HTTP接口配置如下参数 -export PUSH_MODE_HTTP_WORKERS="1" # HTTP服务进程数，在 PUSH_MODE_HTTP_PORT 配置的情况下有效，最高设置到8即可，默认为1 -``` - -更多请求参数请参考[模型配置参数介绍](#模型配置参数介绍) - -### 启动服务 - -``` -cd /opt/output/Serving -bash start_server.sh - -# 重新启动服务前，需要停止服务，在/opt/output/Serving目录下执行 bash stop_server.sh -``` - -### 服务状态查询 - -``` -# port为上面启动服务时候指定的HTTP_PORT - > 测试前请确保服务IP和端口正确 - -live接口： (服务是否能正常接收请求） - http://127.0.0.1:8110/v2/health/live -health接口：（模型是否准备好推理） - http://127.0.0.1:8110/v2/health/ready -``` - -## 服务测试 - -### Python 客户端 - -``` -from fastdeploy_client.chatbot import ChatBot - -hostname = "127.0.0.1" # 服务部署的hostname -port = 8811 # 服务配置的GRPC_PORT - -chatbot = ChatBot(hostname=hostname, port=port) - -# 非流式接口 -result = chatbot.generate("hello", topp=0.8, max_dec_len=128, timeout=120) -print(result) - -# 流式接口 -chatbot = ChatBot(hostname=hostname, port=port) -stream_result = chatbot.stream_generate("hello", max_dec_len=128, timeout=120) -for res in stream_result: - print(res) -``` - -### HTTP调用 - -提示：HTTP调用接口使用变量 PUSH_MODE_HTTP_PORT 配置！HTTP_PORT 仅用于探活接口使用！ - -``` -import uuid -import json -import requests - -push_mode_http_port = "9965" # 服务配置的PUSH_MODE_HTTP_PORT -url = f"http://127.0.0.1:{push_mode_http_port}/v1/chat/completions" -req_id = str(uuid.uuid1()) -data_single = { - "text": "Hello, how are you?", - "req_id": req_id, - "max_dec_len": 64, - "stream": True, - } -# 逐token返回 -res = requests.post(url, json=data_single, stream=True) -for line in res.iter_lines(): - print(json.loads(line)) - -# 多轮对话 -data_multi = { - "messages": [ - {"role": "user", "content": "Hello, who are you"}, - {"role": "system", "content": "I'm a helpful AI assistant."}, - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - "req_id": req_id, - "max_dec_len": 64, - "stream": True, - } -# 逐token返回 -res = requests.post(url, json=data_multi, stream=True) -for line in res.iter_lines(): - print(json.loads(line)) -``` - -更多请求参数请参考[请求参数介绍](#请求参数介绍) - -### 返回示例 - -``` -如果stream为True，流式返回 - 如果正常，返回{'token': xxx, 'is_end': xxx, 'send_idx': xxx, ..., 'error_msg': '', 'error_code': 0} - 如果异常，返回{'error_msg': xxx, 'error_code': xxx}，error_msg字段不为空，error_code字段不为0 - -如果stream为False，非流式返回 - 如果正常，返回{'tokens_all': xxx, ..., 'error_msg': '', 'error_code': 0} - 如果异常，返回{'error_msg': xxx, 'error_code': xxx}，error_msg字段不为空，error_code字段不为0 -``` - -### OpenAI 客户端 - -我们提供了 OpenAI 客户端的支持，使用方法如下： - -提示：使用 OpenAI 客户端需要配置 `PUSH_MODE_HTTP_PORT`！ - -``` -import openai - -push_mode_http_port = "9965" # 服务配置的PUSH_MODE_HTTP_PORT -client = openai.Client(base_url=f"http://127.0.0.1:{push_mode_http_port}/v1/chat/completions", api_key="EMPTY_API_KEY") - -# 非流式返回 -response = client.completions.create( - model="default", - prompt="Hello, how are you?", - max_tokens=50, - stream=False, -) - -print(response) -print("\n") - -# 流式返回 -response = client.completions.create( - model="default", - prompt="Hello, how are you?", - max_tokens=100, - stream=True, -) - -for chunk in response: - if chunk.choices[0] is not None: - print(chunk.choices[0].text, end='') -print("\n") - -# Chat completion -# 非流式返回 -response = client.chat.completions.create( - model="default", - messages=[ - {"role": "user", "content": "Hello, who are you"}, - {"role": "system", "content": "I'm a helpful AI assistant."}, - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, - stream=False, -) - -print(response) -print("\n") - -# 流式返回 -response = client.chat.completions.create( - model="default", - messages=[ - {"role": "user", "content": "Hello, who are you"}, - {"role": "system", "content": "I'm a helpful AI assistant."}, - {"role": "user", "content": "List 3 countries and their capitals."}, - ], - temperature=0, - max_tokens=64, - stream=True, -) - -for chunk in response: - if chunk.choices[0].delta is not None: - print(chunk.choices[0].delta.content, end='') -print("\n") -``` - -## 基于dockerfile创建自己的镜像 - -为了方便用户构建自定义服务，我们提供了基于dockerfile创建自己的镜像的脚本。 -``` -git clone https://github.com/PaddlePaddle/FastDeploy.git -cd FastDeploy/llm - -docker build --network=host -f ./dockerfiles/Dockerfile_serving_cuda123_cudnn9 -t llm-serving-cu123-self . -``` -创建自己的镜像后，可以基于该镜像[创建容器](#创建容器) - -## 模型配置参数介绍 - -| 字段名 | 字段类型 | 说明 | 是否必填 | 默认值 | 备注 | -| :---: | :-----: | :---: | :---: | :-----: | :----: | -| MP_NUM | int | 模型并行度 | 否 | 8 | CUDA_VISIBLE_DEVICES 需配置对应卡数 | -| CUDA_VISIBLE_DEVICES | str | 使用 GPU 编号 | 否 | 0,1,2,3,4,5,6,7 | | -| HTTP_PORT | int | 探活服务的http端口 | 是 | 无 | 当前仅用于健康检查、探活 | -| GRPC_PORT | int | 模型推服务的grpc端口 | 是 | 无 | | -| METRICS_PORT | int | 模型服务中监督指标的端口 | 是 | 无 | | -| INFER_QUEUE_PORT | int | 模型服务内部使用的端口 | 否 | 56666 | | -| PUSH_MODE_HTTP_PORT | int | 服务请求HTTP端口号 | 否 | -1 | 如不配置，服务只支持GRPC协议 | -| DISABLE_STREAMING | int | 是否使用流式返回 | 否 | 0 | | -| MAX_SEQ_LEN | int | 最大输入序列长度 | 否 | 8192 | 服务会拒绝input token数量超过MAX_SEQ_LEN的请求，并返回错误提示 | -| MAX_DEC_LEN | int | 最大decoer序列长度 | 否 | 1024 | 服务会拒绝请求中max_dec_len/min_dec_len超过此参数的请求，并返回错误提示 | -| BATCH_SIZE | int | 最大Batch Size | 否 | 50 | 模型可同时并发处理的最大输入数量，不能高于128 | -| BLOCK_BS | int | 缓存Block支持的最大Query Batch Size | 否 | 50 | 如果出现out of memeory 错误，尝试减少该数值 | -| BLOCK_RATIO | float | | 否 | 0.75 | 建议配置 输入平均Token数/（输入+输出平均Token数) | -| MAX_CACHED_TASK_NUM | int | 服务缓存队列最大长度 | 否 | 128 | 队列达到上限后，会拒绝新的请求 | -| PUSH_MODE_HTTP_WORKERS | int | HTTP服务进程数 | 否 | 1 | 在 PUSH_MODE_HTTP_PORT 配置的情况下有效，高并发下提高该数值，建议最高配置为8 | -| USE_WARMUP | int | 是否进行 warmup | 否 | 0 | | -| USE_HF_TOKENIZER | int | 是否进行使用huggingface的词表 | 否 | 0 | | -| USE_CACHE_KV_INT8 | int | 是否将INT8配置为KV Cache的类型 | 否 | 0 | c8量化模型需要配置为1 | -| MODEL_DIR | str | 模型文件路径 | 否 | /models/ | | -| FD_MODEL_CONFIG_PATH | str | 模型config文件路径 | 否 | ${model_dir}/config.json | | -| DISTRIBUTED_CONFIG | str | 模型分布式配置文件路径 | 否 | ${model_dir}/rank_mapping.csv | | - -## 请求参数介绍 - -| 字段名 | 字段类型 | 说明 | 是否必填 | 默认值 | 备注 | -| :---: | :-----: | :---: | :---: | :-----: | :----: | -| req_id | str | 请求ID，用于标识一个请求。建议设置req_id，保证其唯一性 | 否 | 随机id | 如果推理服务中同时有两个相同req_id的请求，会返回req_id重复的错误信息 | -| text | str | 请求的文本 | 否 | 无 | text 和 messages 必须有一个 | -| messages | str | 多轮对话文本 | 否 | 无 | 多轮对话以list方式存储 | -| max_dec_len | int | 最大生成token的长度，如果请求的文本token长度加上max_dec_len大于模型的max_seq_len，会返回长度超限的错误信息 | 否 | max_seq_len减去文本token长度 | | -| min_dec_len | int | 最小生成token的长度，最小是1 | 否 | 1 | | -| topp | float | 控制随机性参数，数值越大则随机性越大，范围是0~1 | 否 | 0.7 | | -| temperature | float | 控制随机性参数，数值越小随机性越大，需要大于 0 | 否 | 0.95 | | -| frequency_score | float | 频率分数 | 否 | 0 | | -| penalty_score | float | 惩罚分数 | 否 | 1 | | -| presence_score | float | 存在分数 | 否 | 0 | | -| stream | bool | 是否流式返回 | 否 | False | | -| return_all_tokens | bool | 是否一次性返回所有结果 | 否 | False | 与stream参数差异见表后备注 | -| timeout | int | 请求等待的超时时间，单位是秒 | 否 | 300 | | -| return_usage | bool | 是否返回输入、输出 token 数量 | 否 | False | | - -* 在正确配置PUSH_MODE_HTTP_PORT字段下，服务支持 GRPC 和 HTTP 两种请求服务 - * stream 参数仅对 HTTP 请求生效 - * return_all_tokens 参数对 GRPC 和 HTTP 请求均有效 diff --git a/services/llm/requirements-dev.txt b/services/llm/requirements-dev.txt deleted file mode 100644 index e1eec92d201..00000000000 --- a/services/llm/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ -black[jupyter] == 23.3.0 -isort == 5.11.5 -pre-commit diff --git a/services/llm/server/config/config.pbtxt b/services/llm/server/config/config.pbtxt deleted file mode 100644 index 375c41d0133..00000000000 --- a/services/llm/server/config/config.pbtxt +++ /dev/null @@ -1,20 +0,0 @@ -backend: "python" -max_batch_size: 0 -model_transaction_policy { - decoupled: True -} -input [ - { - name: "IN" - data_type: TYPE_STRING - dims: [ 1 ] - } -] -output [ - { - name: "OUT" - data_type: TYPE_STRING - dims: [ 1 ] - } -] -instance_group [{ kind: KIND_CPU }] diff --git a/services/llm/server/requirements.txt b/services/llm/server/requirements.txt deleted file mode 100644 index d7bd2b1ac6e..00000000000 --- a/services/llm/server/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -# model server -sentencepiece -pycryptodome -tritonclient[all]==2.41.1 -opencv-python -patchify -transformers - -# http server -fastapi -httpx -openai==1.44.1 -asyncio -uvicorn -shortuuid - -# parameter search -pynvml - -# paddlenlp -tiktoken diff --git a/services/llm/server/scripts/start_server.sh b/services/llm/server/scripts/start_server.sh deleted file mode 100644 index e7975b3e838..00000000000 --- a/services/llm/server/scripts/start_server.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/bash - -export GLOG_v=0 -export GLOG_logtostderr=1 -export PYTHONIOENCODING=utf8 -export LC_ALL=C.UTF-8 - -# PaddlePaddle environment variables -export FLAGS_allocator_strategy=auto_growth -export FLAGS_dynamic_static_unified_comm=0 -export FLAGS_use_xqa_optim=1 -export FLAGS_gemm_use_half_precision_compute_type=0 -export NVIDIA_TF32_OVERRIDE=0 - -# Model hyperparameters -export MP_NUM=${MP_NUM:-"1"} # Number of GPUs -export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-"0"} # GPU ids -export MAX_SEQ_LEN=${MAX_SEQ_LEN:-"8192"} -export MAX_DEC_LEN=${MAX_DEC_LEN:-"2048"} -export BATCH_SIZE=${BATCH_SIZE:-"20"} -export BLOCK_BS=${BLOCK_BS:-"4"} -export BLOCK_SIZE=${BLOCK_SIZE:-"64"} -export DTYPE=${DTYPE:-"bfloat16"} -export USE_CACHE_KV_INT8=${USE_CACHE_KV_INT8:-"0"} # c8 model requires configuration 1 -export BLOCK_RATIO=${BLOCK_RATIO:-"0.75"} -export ENC_DEC_BLOCK_NUM=${ENC_DEC_BLOCK_NUM:-"4"} -export FIRST_TOKEN_ID=${FIRST_TOKEN_ID:-"1"} -export MAX_PREFILL_BATCH=${MAX_PREFILL_BATCH:-"4"} -export STOP_THRESHOLD=${STOP_THRESHOLD:-"0"} -export MODEL_DIR=${MODEL_DIR:-"/models"} -export DISTRIBUTED_CONFIG=${DISTRIBUTED_CONFIG:-"${MODEL_DIR}/rank_mapping.csv"} -export CONFIG_JSON_FILE=${CONFIG_JSON_FILE:-"config.json"} -export PUSH_MODE_HTTP_WORKERS=${PUSH_MODE_HTTP_WORKERS:-"4"} - -# serving port -export HTTP_PORT=${HTTP_PORT:-"8110"} -export GRPC_PORT=${GRPC_PORT:-"8811"} -export METRICS_PORT=${METRICS_PORT:-"8722"} -export INFER_QUEUE_PORT=${INFER_QUEUE_PORT:-"8813"} -export PUSH_MODE_HTTP_PORT=${PUSH_MODE_HTTP_PORT:-"9965"} - -mkdir -p log -rm -rf console.log log/* -rm -rf /dev/shm/* - -echo "start serving ..." - -tritonserver --exit-timeout-secs 100 --cuda-memory-pool-byte-size 0:0 --cuda-memory-pool-byte-size 1:0 \ - --cuda-memory-pool-byte-size 2:0 --cuda-memory-pool-byte-size 3:0 --cuda-memory-pool-byte-size 4:0 \ - --cuda-memory-pool-byte-size 5:0 --cuda-memory-pool-byte-size 6:0 --cuda-memory-pool-byte-size 7:0 \ - --pinned-memory-pool-byte-size 0 --model-repository llm_model/ \ - --allow-http false \ - --grpc-port=${GRPC_PORT} \ - --metrics-port=${METRICS_PORT} \ - --log-file log/server.log --log-info true > log/console.log 2>&1 & - -echo "The logs for the model service, please check" ${PWD}"/log/server.log and "${PWD}"/log/workerlog.0" diff --git a/services/llm/server/scripts/stop_server.sh b/services/llm/server/scripts/stop_server.sh deleted file mode 100644 index 89cfa42f3aa..00000000000 --- a/services/llm/server/scripts/stop_server.sh +++ /dev/null @@ -1,68 +0,0 @@ -# /bin/bash - -pids=($(ps aux | grep -E 'tritonserver' | grep -v grep | awk '{print $2}')) - -if [ ${#pids[@]} -eq 0 ]; then - echo "Can not find tritonserver." - timeout=1 -else - timeout=300 -fi - -# kill processor -for pid in "${pids[@]}"; do - echo "killing $pid" - kill -2 "$pid" -done - -timeout_interval=$1 -if [ ! "$timeout_interval" == "" ]; then - timeout=$timeout_interval - echo $timeout -fi - -start_time=$(date +%s) - -while : ; do - current_time=$(date +%s) - - elapsed_time=$((current_time - start_time)) - - if [ $elapsed_time -ge $timeout ]; then - echo "forcibly kill all process ..." - pids=$(ps auxww | grep -E "tritonserver|triton_python_backend_stub|infer|multiprocessing.resource_tracker|paddle.distributed.launch|task_queue_manager|app.py|spawn_main" | grep -v grep | grep -v start_both | awk '{print $2}'); - echo $pids; - for pid in ${pids[@]}; do - kill -9 ${pid} - done - break - fi - - pids=$(ps auxww | grep -E "tritonserver|triton_python_backend_stub|multiprocessing.resource_tracker|paddle.distributed.launch|app.py|spawn_main" | grep -v grep | awk '{print $2}'); - array=($(echo "$pids" | tr ' ' '\n')) - - if [ ${#array[*]} -ne 0 ]; then - echo "cleaning process, please wait ..." - sleep 1 - else - echo "clean finished." - break - fi -done - -manager_pids=$(ps auxww | grep "task_queue_manager" | grep -v grep | awk '{print $2}') -echo $manager_pids -for in_pid in ${manager_pids[@]}; do - kill -9 ${in_pid} -done -echo 'end kill queue manager' - -health_checker_pids=$(ps auxww | grep "health.py" | grep -v grep | awk '{print $2}') -echo $health_checker_pids -for in_pid in ${health_checker_pids[@]}; do - kill -9 ${in_pid} -done -echo 'end kill health checker' - -echo "all process terminated." -exit 0 diff --git a/services/llm/server/server/__init__.py b/services/llm/server/server/__init__.py deleted file mode 100644 index 5ae9b7e8cf1..00000000000 --- a/services/llm/server/server/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version__ = "dev" -__commit__ = "dev" diff --git a/services/llm/server/server/checker.py b/services/llm/server/server/checker.py deleted file mode 100644 index e9f776799df..00000000000 --- a/services/llm/server/server/checker.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def check_basic_params(req_dict): - """ - checks input requests for basic parameters - - Args: - req_dict (dict): request parameters - - Returns: - list[str]: if error, return a list of error messages; return an empty list otherwise - """ - - error_msg = [] - bools = ("text" in req_dict, "input_ids" in req_dict, "messages" in req_dict) - if sum(bools) == 0: - error_msg.append("The input parameters should contain either `text`, `input_ids` or `messages`") - else: - if "text" in req_dict: - if not isinstance(req_dict["text"], str): - error_msg.append("The `text` in input parameters must be a string") - elif req_dict["text"] == "": - error_msg.append("The `text` in input parameters cannot be empty") - if "system" in req_dict and not isinstance(req_dict["system"], str): - error_msg.append("The `system` in input parameters must be a string") - if "input_ids" in req_dict and not isinstance(req_dict["input_ids"], list): - error_msg.append("The `input_ids` in input parameters must be a list") - if "messages" in req_dict: - msg_len = len(req_dict["messages"]) - if not all("content" in item for item in req_dict["messages"]): - error_msg.append("The item in messages must include `content`") - - if "req_id" not in req_dict: - error_msg.append("The input parameters should contain `req_id`.") - - if "min_dec_len" in req_dict and \ - (not isinstance(req_dict["min_dec_len"], int) or req_dict["min_dec_len"] < 1): - error_msg.append("The `min_dec_len` must be an integer and greater than 0") - - keys = ("max_dec_len", "seq_len", "max_tokens") - for key in keys: - if key in req_dict and (not isinstance(req_dict[key], int) or req_dict[key] < 1): - error_msg.append(f"The `{key}` must be an integer and greater than 0") - if "seq_len" in req_dict and "max_dec_len" not in req_dict: - req_dict["max_dec_len"] = req_dict["seq_len"] - if "max_tokens" in req_dict and "max_dec_len" not in req_dict: - req_dict["max_dec_len"] = req_dict["max_tokens"] - - keys = ("topp", "top_p") - if sum([key in req_dict for key in keys]) > 1: - error_msg.append(f"Only one of {keys} should be set") - else: - for key in keys: - if key in req_dict and not 0 <= req_dict[key] <= 1: - error_msg.append(f"The `{key}` must be in [0, 1]") - if "top_p" in req_dict and "topp" not in req_dict: - req_dict["topp"] = req_dict["top_p"] - - if "temperature" in req_dict and not 0 <= req_dict["temperature"]: - error_msg.append(f"The `temperature` must be >= 0") - - if "eos_token_ids" in req_dict: - if isinstance(req_dict["eos_token_ids"], int): - req_dict["eos_token_ids"] = [req_dict["eos_token_ids"]] - elif isinstance(req_dict["eos_token_ids"], tuple): - req_dict["eos_token_ids"] = list(req_dict["eos_token_ids"]) - if not isinstance(req_dict["eos_token_ids"], list): - error_msg.append("The `eos_token_ids` must be an list") - elif len(req_dict["eos_token_ids"]) > 1: - error_msg.append("The length of `eos_token_ids` must be 1 if you set it") - - keys = ("infer_seed", "seed") - if sum([key in req_dict for key in keys]) > 1: - error_msg.append(f"Only one of {keys} should be set") - else: - if "seed" in req_dict and "infer_seed" not in req_dict: - req_dict["infer_seed"] = req_dict["seed"] - - if "stream" in req_dict and not isinstance(req_dict["stream"], bool): - error_msg.append("The `stream` must be a boolean") - - if "response_type" in req_dict and (req_dict["response_type"].lower() not in ("fastdeploy", "openai")): - error_msg.append("The `response_type` must be either `fastdeploy` or `openai`.") - - return error_msg - - -def add_default_params(req_dict): - """ - add default params to req_dict - - Args: - req_dict (dict): input dict - - Returns: - dict: req_dict with default params - """ - assert isinstance(req_dict, dict), "The `req_dict` must be a dict." - if "min_dec_len" not in req_dict: - req_dict["min_dec_len"] = 1 - if "topp" not in req_dict: - req_dict["topp"] = 0.7 - if "temperature" not in req_dict: - req_dict["temperature"] = 0.95 - if "penalty_score" not in req_dict: - req_dict["penalty_score"] = 1.0 - if "frequency_score" not in req_dict: - req_dict["frequency_score"] = 0.0 - if "presence_score" not in req_dict: - req_dict["presence_score"] = 0.0 - return req_dict diff --git a/services/llm/server/server/data/__init__.py b/services/llm/server/server/data/__init__.py deleted file mode 100644 index 97043fd7ba6..00000000000 --- a/services/llm/server/server/data/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/services/llm/server/server/data/processor.py b/services/llm/server/server/data/processor.py deleted file mode 100644 index 175f33b2db1..00000000000 --- a/services/llm/server/server/data/processor.py +++ /dev/null @@ -1,389 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from abc import ABC, abstractmethod - -from paddlenlp.transformers import Llama3Tokenizer, LlamaTokenizer -from paddlenlp.trl.llm_utils import get_eos_token_id -from server.engine.config import Config -from server.utils import data_processor_logger - - -class BaseDataProcessor(ABC): - """base class for data processor""" - - def __init__(self): - """ - Returns: - None - """ - self.tokenizer = self._load_tokenizer() - self.tokenizer.bos_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.bos_token) - self.tokenizer.cls_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.cls_token) - self.tokenizer.sep_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.sep_token) - self.tokenizer.eos_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.eos_token) - self.tokenizer.mask_token_id = self.tokenizer._convert_token_to_id(self.tokenizer.mask_token) - data_processor_logger.info((f"tokenizer infomation: bos_token is {self.tokenizer.bos_token}, {self.tokenizer.bos_token_id}, ", - f"cls_token is {self.tokenizer.cls_token}, {self.tokenizer.cls_token_id}, " - f"sep_token is {self.tokenizer.sep_token}, {self.tokenizer.sep_token_id}, " - f"eos_token is {self.tokenizer.eos_token}, {self.tokenizer.eos_token_id}, " - f"mask_token is {self.tokenizer.mask_token}, {self.tokenizer.mask_token_id}")) - - @abstractmethod - def process_request(self, request, **kwargs): - """ - Preprocess the request - - Args: - request (Dict): may contain text and messages fields - **kwargs: others - - Returns: - bool: Whether preprocessing is successful - str: error message - """ - raise NotImplementedError - - @abstractmethod - def process_response(self, response_dict): - """ - Preprocess the response - - Args: - response_dict (Dict): response for engine, contain ids fields - - Returns: - Dict: response contain text fields - """ - raise NotImplementedError - - def text2ids(self, text): - """ - text to token ids - - Args: - text (str): text - - Returns: - List[int]: token ids list - """ - raise NotImplementedError - - def messages2ids(self, messages): - """ - Convert multi-turn messages into ID sequences. - - Args: - messages (List[List[Dict[str, Any]]]): multi-turn messages. - - Returns: - List[int]: ID sequences - """ - raise NotImplementedError - - def ids2tokens(self, token_ids, task_id=None): - """ - token ids to strings - - Args: - token_ids (List[int]): token ids - task_id (str): task id - - Returns: - List[str]: strings - """ - raise NotImplementedError - - @abstractmethod - def _load_tokenizer(self): - """ - load tokenizer - - Returns: - tokenizer (AutoTokenizer) - """ - raise NotImplementedError - - -class DataProcessor(BaseDataProcessor): - def __init__(self): - self.config = Config() - max_length = self.config.get_model_config().get('max_length', 1024) - self.src_length = max_length - self.config.seq_len_limit - - self.decode_status = dict() - self.tokenizer = self._load_tokenizer() - data_processor_logger.info(f"tokenizer infomation: bos_token is {self.tokenizer.bos_token}, {self.tokenizer.bos_token_id}, \ - eos_token is {self.tokenizer.eos_token}, {self.tokenizer.eos_token_id} ") - - def process_request(self, request, max_seq_len=None): - """ - Preprocess the request - - Args: - request (Dict): may contain text and messages fields - - Returns: - bool: Whether preprocessing is successful - str: error message - """ - if "eos_token_ids" not in request or request["eos_token_ids"] == [None]: - request["eos_token_ids"] = [] - request["eos_token_ids"].extend(get_eos_token_id(self.tokenizer, self.config.generation_config)) - - if "stop_seqs" not in request or (isinstance(request["stop_seqs"], (list, tuple)) and len(request["stop_seqs"]) == 0): - self.update_stop_seq(request) - - if "input_ids" not in request or \ - (isinstance(request["input_ids"], (list, tuple)) and len(request["input_ids"]) == 0): - if "text" in request: - request["input_ids"] = self.text2ids(request["text"]) - elif "messages" in request: - if self.tokenizer.chat_template is None: - raise ValueError(f"This model does not support chat_template.") - request["input_ids"] = self.messages2ids(request["messages"]) - else: - raise ValueError(f"The request should have `input_ids`, `text` or `messages`: {request}.") - - if max_seq_len is not None and len(request["input_ids"]) > max_seq_len: - request["input_ids"] = request["input_ids"][:max_seq_len-1] - data_processor_logger.info(f"processed request: {request}") - return request - - def process_response(self, response_dict, **kwargs): - """ - Preprocess the response - - Args: - response_dict (Dict): response for engine, contain ids fields - - Returns: - Dict: response contain text fields - """ - is_end = response_dict.get("is_end", 0) - req_id = response_dict.get("req_id") - if "choices" in response_dict: - for i in range(len(response_dict["choices"])): - response_dict["token"] = self.ids2tokens(response_dict["choices"][i]["token_ids"], req_id) - return response_dict - - token_ids = response_dict.get("token_ids", []) - response_dict["token"] = self.ids2tokens(token_ids, response_dict["req_id"]) - response_dict["usage"] = {"completion_tokens" : response_dict["send_idx"] + 1} - - if is_end: - response_dict["tokens_all"] = self.clear_request_status(req_id) - return response_dict - - def text2ids(self, text): - """ - text to token ids - - Args: - text (str): text - - Returns: - List[int]: token ids list - """ - if self.config.use_hf_tokenizer: - tokens = self.tokenizer( - text, - return_tensors="np", - padding=True, - truncation=True, - ) - else: - if self.tokenizer.chat_template is not None: - text = [text] if isinstance(text, str) else text - text = [self.tokenizer.apply_chat_template(sentence, tokenize=False) for sentence in text] - - tokens = self.tokenizer( - text, - return_tensors="np", - padding=True, - truncation=True, - max_length=self.src_length, - add_special_tokens=self.tokenizer.chat_template is None, - ) - return tokens["input_ids"][0] - - def messages2ids(self, messages): - """ - Convert multi-turn messages into ID sequences. - - Args: - messages (List[List[Dict[str, Any]]]): multi-turn messages. - - Returns: - List[int]: ID sequences - """ - message_result = self.tokenizer.apply_chat_template(messages, return_tensors="pd") - return message_result["input_ids"][0] - - def ids2tokens(self, token_id, task_id): - """ - token ids to strings - - Args: - token_ids (List[int]): token ids - task_id (str): task id - - Returns: - List[str]: strings - """ - if self.config.use_hf_tokenizer: - if task_id not in self.decode_status: - # history token ids & history token strings & befer decode str - self.decode_status[task_id] = [[], [], ""] - - previous_token_ids = self.decode_status[task_id][0] - decode_str = self.tokenizer.batch_decode([previous_token_ids + token_id], - skip_special_tokens=True, - clean_up_tokenization_spaces=False) - if isinstance(decode_str, list) and len(decode_str): - new_str = decode_str[0].replace(self.decode_status[task_id][2], "", 1) - self.decode_status[task_id][1].append(new_str) - self.decode_status[task_id][2] = decode_str[0] - else: - new_str = "" - self.decode_status[task_id][0] += token_id - return new_str - else: - if task_id not in self.decode_status: - # prefix offset & read offset & history token ids & history token strings - self.decode_status[task_id] = [0, 0, [], []] - - prefix_offset = self.decode_status[task_id][0] - read_offset = self.decode_status[task_id][1] - previous_token_ids = self.decode_status[task_id][2] - decode_str, prefix_offset, read_offset = self.tokenizer.decode_token( - previous_token_ids + token_id, prefix_offset, read_offset) - self.decode_status[task_id][0] = prefix_offset - self.decode_status[task_id][1] = read_offset - self.decode_status[task_id][2] += token_id - self.decode_status[task_id][3].append(decode_str) - return decode_str - - def _load_tokenizer(self): - """ - load tokenizer - - Returns: - tokenizer (AutoTokenizer) - """ - if self.config.use_hf_tokenizer: - from transformers import AutoTokenizer - return AutoTokenizer.from_pretrained(self.config.model_dir, use_fast=False) - else: - from paddlenlp.transformers import AutoTokenizer - return AutoTokenizer.from_pretrained(self.config.model_dir) - - def clear_request_status(self, task_id): - """ - clear request status - - Args: - task_id (str): task id - - Returns: - results_all (str): all token strings - """ - results_all = "" - if task_id in self.decode_status: - if self.config.use_hf_tokenizer: - results_all = self.decode_status[task_id][2] - else: - results_all = "".join(self.decode_status[task_id][3]) - del self.decode_status[task_id] - return results_all - - def get_eos_tokens_lens(self): - """ - get eos_token_id lens - - Returns: - int: eos_token_id lens - """ - return len(get_eos_token_id(self.tokenizer, self.config.generation_config)) - - def get_eos_tokens(self): - """ - get all eos_token_id - - Returns: - List[int]: eos_token_id list - """ - return get_eos_token_id(self.tokenizer, self.config.generation_config) - - def get_pad_id(self): - """ - get pad_token_id, if not pad_token_id, use eos_token - - Returns: - int: pad_token_id - """ - if isinstance(self.tokenizer, (LlamaTokenizer, Llama3Tokenizer)) and not self.tokenizer.pad_token_id: - return self.tokenizer.eos_token - return self.tokenizer.pad_token_id - - def pad_batch_data(self, insts, pad_id=0, return_seq_len=False, return_array=True, pad_style="right"): - """Pad the instances to the max sequence length in batch.""" - if len(insts) == 0: - padded_insts = np.array([[]], dtype=np.int64) if return_array else [[]] - if return_seq_len: - seq_len = np.array([], dtype=np.int64) if return_array else [] - return padded_insts, seq_len - return padded_insts - - max_len = max(map(len, insts)) - if pad_style == "left": - padded_insts = [[pad_id] * (max_len - len(inst)) + list(inst) for inst in insts] - else: - padded_insts = [list(inst) + [pad_id] * (max_len - len(inst)) for inst in insts] - if return_array: - padded_insts = np.array(padded_insts, dtype=np.int64).reshape([-1, max_len]) - - if return_seq_len: - seq_len = [len(inst) for inst in insts] - if return_array: - seq_len = np.array(seq_len, dtype=np.int64).reshape(-1, 1) - return padded_insts, seq_len - return padded_insts - - def update_stop_seq(self, request): - """ - Update stop sequences from request. - """ - max_stop_seqs_num = int(os.getenv("MAX_STOP_SEQS_NUM", 5)) - stop_seqs_max_len = int(os.getenv("STOP_SEQS_MAX_LEN", 8)) - stop_seqs = [] - raw_stop_sequences = request.get("stop_sequences", []) or [] - for seq in raw_stop_sequences[:max_stop_seqs_num]: - if seq != self.tokenizer.eos_token_id: - token_ids = self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(seq)) - if len(token_ids) > stop_seqs_max_len: - token_ids = token_ids[:stop_seqs_max_len] - stop_seqs.append(token_ids) - if len(stop_seqs) == 0: - request["stop_seqs"] = [] - request["stop_seqs_len"] = [] - else: - request["stop_seqs"], request["stop_seqs_len"] = self.pad_batch_data( - stop_seqs, - pad_id=-1, - return_seq_len=True, - return_array=False - ) - data_processor_logger.debug(f"processed request: {request['stop_seqs'], request['stop_seqs_len']}") diff --git a/services/llm/server/server/engine/__init__.py b/services/llm/server/server/engine/__init__.py deleted file mode 100644 index fd05a920816..00000000000 --- a/services/llm/server/server/engine/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/services/llm/server/server/engine/config.py b/services/llm/server/server/engine/config.py deleted file mode 100644 index 3b9a88f0c94..00000000000 --- a/services/llm/server/server/engine/config.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import sys -from datetime import datetime - -from paddlenlp.generation import GenerationConfig -from server.utils import model_server_logger -from dataclasses import dataclass - - -class Config: - """ - initial configuration - """ - - def __init__(self): - self.read_from_env() - - def read_from_env(self): - """ - get the configuration from environment - """ - env = os.environ - self.model_dir = env.get( - "MODEL_DIR", "/opt/output/Serving/models") - if not self.model_dir: - raise Exception("The parameter MODEL_DIR is None.") - self.mp_num = int(env.get("MP_NUM", 8)) - self.config_json_file = env.get("CONFIG_JSON_FILE", "config.json") - self.model_config_path = os.path.join(self.model_dir, self.config_json_file) - if env.get("FD_MODEL_CONFIG_PATH", None): - self.model_config_path = env.get("FD_MODEL_CONFIG_PATH") - - # distributed config - self.distributed_config_path = os.path.join(self.model_dir, "rank_mapping.csv") - if os.getenv("DISTRIBUTED_CONFIG", None): - self.distributed_config_path = os.getenv("DISTRIBUTED_CONFIG") - - # device config - self.device = env.get("DEVICE", "GPU") - self.device_ids = ",".join([str(i) for i in range(self.mp_num)]) - if self.device == "GPU": - self.device_ids = os.getenv("CUDA_VISIBLE_DEVICES", - self.device_ids) - else: - raise Exception(f"unsupported device type: {self.device}") - - # Triton config - self.max_prefill_batch = int(os.getenv("MAX_PREFILL_BATCH", 1)) - if self.max_prefill_batch <= 0: - raise Exception(f"MAX_PREFILL_BATCH ({self.max_prefill_batch}) must be greater than 0") - self.disable_streaming = int(os.getenv("DISABLE_STREAMING", 0)) - - # max cached task num - self.max_cached_task_num = int(os.getenv("MAX_CACHED_TASK_NUM", "128")) - # if PUSH_MODE_HTTP_PORT is not configured, only GRPC service is enabled - self.push_mode_http_port = int(os.getenv("PUSH_MODE_HTTP_PORT", "-1")) - if self.push_mode_http_port > 0: - grpc_port = os.getenv("GRPC_PORT", None) - if grpc_port is None: - raise Exception("GRPC_PORT cannot be None, while PUSH_MODE_HTTP_PORT>0") - self.grpc_port = int(grpc_port) - - # http worker num - self.push_mode_http_workers = int(os.getenv("PUSH_MODE_HTTP_WORKERS", "1")) - if self.push_mode_http_workers < 1: - raise Exception(f"PUSH_MODE_HTTP_WORKERS ({self.push_mode_http_workers}) must be positive") - - # Padlle commit id - import paddle - self.paddle_commit_id = paddle.version.commit - - # time interval for detecting whether the engine loop is normal during probing - self.check_health_interval = int(os.getenv("CHECK_HEALTH_INTERVAL", 10)) - - # model config - self.dtype = env.get("DTYPE", "bfloat16") - self.block_size = int(env.get("BLOCK_SIZE", 64)) - self.use_cache_kv_int8 = int(os.getenv("USE_CACHE_KV_INT8", 0)) - self.use_cache_kv_int4 = int(os.getenv("USE_CACHE_KV_INT4", 0)) - - # infer config - self.max_batch_size = int(env.get("BATCH_SIZE", 50)) - self.max_seq_len = int(env.get("MAX_SEQ_LEN", 8192)) - self.max_dec_len = int(env.get("MAX_DEC_LEN", 1024)) - self.enc_dec_block_num = int(os.getenv("ENC_DEC_BLOCK_NUM", 2)) - self.block_bs = float(env.get("BLOCK_BS", 50)) - self.block_ratio = float(os.getenv("BLOCK_RATIO", 0.75)) - self.bad_tokens = str(env.get("BAD_TOKENS", "-1")) - self.first_token_id = int(os.getenv("FIRST_TOKEN_ID", 1)) - - # infer queue port - self.infer_port = int(os.getenv("INFER_QUEUE_PORT", 56666)) - - # whether to use custom health checker - self.use_custom_health_checker = int(os.getenv("USE_CUSTOM_HEALTH_CHECKER", 1)) - - # Check the legality of requests - self.seq_len_limit = int(env.get("MAX_SEQ_LEN", 8192)) - self.dec_len_limit = int(env.get("MAX_DEC_LEN", 1024)) - - # warmup - self.use_warmup = int(os.getenv("USE_WARMUP", 0)) == 1 - - # uuid - self.shm_uuid = os.getenv("SHM_UUID", '') - - # use huggingface tokenizer - self.use_hf_tokenizer = int(os.getenv("USE_HF_TOKENIZER", 0)) == 1 - - # Generation config - try: - self.generation_config = GenerationConfig.from_pretrained(self.model_dir) - except: - model_server_logger.warning( - "Can't find generation config, so it will not use generation_config field in the model config" - ) - self.generation_config = None - - self.read_from_config() - self.postprocess() - self.check() - - def postprocess(self): - """ - calculate some parameters - """ - if self.block_ratio >= 1.0: - self.enc_dec_block_num = (self.max_dec_len + self.block_size - 1) // self.block_size - self.max_query_block_num = (max(self.max_dec_len, self.max_seq_len) + - self.block_size - 1) // self.block_size - self.max_query_block_num = (self.max_dec_len + self.max_seq_len + - self.block_size - 1) // self.block_size - self.dec_token_num = self.enc_dec_block_num * self.block_size - self.total_block_num = int(self.block_bs * self.max_query_block_num) - self.max_block_num = int(self.total_block_num * self.block_ratio) - model_server_logger.info(f"max_block_num:{self.max_block_num}") - - def check(self): - """ - check the legality of config - """ - assert self.max_batch_size <= 256, ( - "The parameter `max_batch_size` is not allowed to exceed 256, " - "but now it's {}.".format(self.max_batch_size) - ) - assert self.seq_len_limit <= self.max_seq_len, ( - f"The seq_len_limit shouldn't greater than max_seq_len in model, " - f"which means the exported MAX_SEQ_LEN should less than " - f"{self.max_seq_len}, but now it's {self.seq_len_limit}." - ) - assert self.dec_len_limit <= self.max_seq_len, ( - f"The dec_len_limit shouldn't greater than max_seq_len in model, " - f"which means the exported MAX_DEC_LEN should less than " - f"{self.max_seq_len}, but now it's {self.dec_len_limit}." - ) - - def print(self, file=None): - """ - print all config - - Args: - file (str): the path of file to save config - """ - model_server_logger.info( - "=================== Configuration Information ===============") - for k, v in self.__dict__.items(): - if k == "generation_config" and v is not None: - for gck, gcv in v.to_dict().items(): - model_server_logger.info("{:<20}:{:<6}{}".format(gck, "", gcv)) - else: - model_server_logger.info("{:<20}:{:<6}{}".format(k, "", v)) - model_server_logger.info( - "=============================================================") - if file is not None: - f = open(file, "a") - now_time = datetime.now() - f.write(f"{now_time} configuration information as below,\n") - for k, v in self.__dict__.items(): - f.write("{:<20}:{:<6}{}\n".format(k, "", v)) - f.close() - - def get_model_config(self): - """ - load config file - - Returns: - dict: the config file - """ - model_config_json = json.load(open(self.model_config_path, 'r', encoding='utf-8')) - return model_config_json - - def get_speculate_config(self): - """ - get speculate_decoding related config - - Returns: - SpeculateConfig: the speculate related config - """ - speculate_config = SpeculateConfig() - model_cfg = self.get_model_config() - if model_cfg.get("speculate_method", "None") != "None": - speculate_config.speculate_method = str(model_cfg["speculate_method"]) - speculate_config.speculate_max_draft_token_num = model_cfg[ - "speculate_max_draft_token_num"] - speculate_config.speculate_max_ngram_size = model_cfg[ - "speculate_max_ngram_size"] - - if speculate_config.speculate_method not in ["None", "inference_with_reference"]: - model_server_logger.error(f"Unsupport speculate method: {speculate_config.speculate_method}") - - return speculate_config - - def read_from_config(self): - """ - reset model config from json file - """ - from server.utils import get_logger - logger = get_logger("model_server", "infer_config.log") - config = self.get_model_config() - - def reset_value(self, value_name, key, config): - if key in config: - value = config[key] - setattr(self, value_name, value) - logger.info(f"Reset parameter {value_name} = {value} from configuration.") - - reset_value(self, "block_size", "infer_model_block_size", config) - reset_value(self, "max_seq_len", "infer_model_max_seq_len", config) - - assert self.seq_len_limit <= self.max_seq_len, f"The loading model requires len(input_ids) <= {self.max_seq_len}, but now the setting MAX_SEQ_LEN={self.seq_len_limit}." - assert self.dec_len_limit <= self.max_seq_len, f"The loading model requires MAX_DEC_LEN <= {self.max_seq_len}, but now the setting MAX_DEC_LEN={self.dec_len_limit}." - - def get_unique_name(self, name): - """ - get unique name - - Args: - name (str): the name add uuid - """ - return name + f"_{self.shm_uuid}" - - def __str__(self) -> str: - return json.dumps(self.__dict__, indent=4) - - -@dataclass -class SpeculateConfig: - speculate_method: str = "None" - speculate_max_draft_token_num: int = 1 - speculate_max_ngram_size: int = 1 \ No newline at end of file diff --git a/services/llm/server/server/engine/engine.py b/services/llm/server/server/engine/engine.py deleted file mode 100644 index 932404d9c09..00000000000 --- a/services/llm/server/server/engine/engine.py +++ /dev/null @@ -1,401 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import multiprocessing -import os -import signal -import subprocess -import time -import uuid -import weakref -from datetime import datetime -from multiprocessing import shared_memory - -import numpy as np -from server.engine.resource_manager import ResourceManager -from server.engine.task_queue_manager import (TaskQueueManager, - launch_queue_service) -from server.engine.token_processor import TokenProcessor, WarmUpTokenProcessor -from server.utils import model_server_logger - - -class Engine(object): - """ - Engine Class - """ - def __init__(self, cfg, token_processor): - self.cfg = cfg - self.resource_manager = ResourceManager(self.cfg) - self.token_processor = token_processor - self.token_processor.set_resource_manager(self.resource_manager) - self.is_started = False - - self._init_engine_flags() - self._finalizer = weakref.finalize(self, self._exit_sub_services) - - def start(self): - """ - initialize engine and start sub services - """ - assert not self.is_started, "The engine is already started.!" - start_time = time.time() - self.queue_service = self._start_tasks_queue_service() - self.tasks_queue = TaskQueueManager(mp_num=self.cfg.mp_num, port=self.cfg.infer_port) - - self.token_processor.tasks_queue = self.tasks_queue - self.infer_proc = self._start_infer_service() - model_server_logger.info("Waitting infer processes ready...") - while not self._infer_processes_ready(): - time.sleep(1) - self.is_started = True - - # start warmup - if self.cfg.use_warmup: - model_server_logger.info("Start warmup") - self._set_warmup_token_processor() - self.warmup() - self._del_warmup_token_processor() - model_server_logger.info("Warmup finish") - - # start TokenProcessor thread - self.token_processor.run() - model_server_logger.info("Infer processes are launched with {} seconds.".format(time.time() - start_time)) - - def warmup(self): - """ - construct test tasks and avoid out of memory problem in the infer process - """ - # get eos_token_id - from server.data.processor import DataProcessor - eos_token_ids = DataProcessor().get_eos_tokens() - - # construct test tasks - res_task = [] - for j in range(2 * self.cfg.max_batch_size): - data = { - "input_ids": [5], - "req_id": j, - "max_dec_len": self.cfg.dec_len_limit, - "min_dec_len": int(self.cfg.dec_len_limit * 0.5) + 1, - "eos_token_ids": eos_token_ids - } - res_task.append(data) - for j in range(2 * self.cfg.max_prefill_batch): - data = { - "input_ids": [5] * self.cfg.seq_len_limit, - "req_id": j + 2 * self.cfg.max_batch_size, - "max_dec_len": 1, - "min_dec_len": 1, - "eos_token_ids": eos_token_ids - } - res_task.append(data) - - for x in res_task: - while self.available_batch() == 0 or not self.insert_tasks([x]): - time.sleep(0.0002) - - self.token_processor._is_blocking = False - # wait for all tasks finished - while not self.all_tasks_finished(): - time.sleep(1) - - def insert_tasks(self, tasks): - """ - insert tasks to the engine - - Args: - tasks: list of tasks - - Returns: - return: True if success, False otherwise - """ - if not isinstance(tasks, list): - tasks = [tasks] - - for item in tasks: - item["schedule_start_time"] = datetime.now() - - available_batch = np.sum(self.resource_manager.stop_flags) - if len(tasks) > available_batch: - model_server_logger.error("Inserting batch:{} exceeds the available batch:{}.".format( - len(tasks), available_batch)) - model_server_logger.error("The exceeded part will be ignored!") - tasks = tasks[:available_batch] - - for i in range(len(tasks)): - req_id = tasks[i]["req_id"] - input_token_num = len(tasks[i]["input_ids"]) - if input_token_num >= self.cfg.max_seq_len - 1: - model_server_logger.warning(f"{req_id}: Input length:{input_token_num}, exceed the limits.") - tasks[i]["input_ids"] = tasks[i]["input_ids"][:self.cfg.max_seq_len - 1] - if "seq_len" in tasks[i] and "max_dec_len" not in tasks[i]: - tasks[i]["max_dec_len"] = tasks[i]["seq_len"] - - # max_dec_len + input_token_num > MAX_SEQ_LEN - if input_token_num + tasks[i]["max_dec_len"] > self.cfg.max_seq_len: - tasks[i]["max_dec_len"] = self.cfg.max_seq_len - input_token_num - model_server_logger.warning("Force max_dec_len to be {} for req_id={}.".format( - tasks[i]["max_dec_len"], tasks[i]["req_id"])) - - # min_dec_len + input_token_num > MAX_SEQ_LEN - if input_token_num + tasks[i]["min_dec_len"] > self.cfg.max_seq_len: - tasks[i]["min_dec_len"] = self.cfg.max_seq_len - input_token_num - model_server_logger.warning("Force min_dec_len to be {} for req_id={}.".format( - tasks[i]["min_dec_len"], tasks[i]["req_id"])) - - tasks = self.resource_manager.allocate_resources_for_new_tasks(tasks) - if not tasks: - return False - - self.token_processor.number_of_tasks += len(tasks) - for i in range(len(tasks)): - self.token_processor.number_of_input_tokens += len(tasks[i]["input_ids"]) - - req_ids = [t["req_id"] for t in tasks] - model_server_logger.info(f"Tasks are sent to engine, req_ids={req_ids}") - self.tasks_queue.put((tasks, self.resource_manager.real_bsz)) - return True - - def task_is_finished(self, index): - """ - judge if the task is finished - - Args: - index: task index - - Returns: - return: True if finished, False otherwise - """ - assert index < len(self.resource_manager.stop_flags) - return self.resource_manager.stop_flags[index] - - def is_queue_empty(self): - """ - judge if the queue is empty - - Returns: - return: True if empty, False otherwise - """ - return self.tasks_queue.empty() - - def is_resource_sufficient(self, input_token_num): - """ - judge if the resource is sufficient - - Args: - input_token_num: input token number - - Returns: - return: True if sufficient, False otherwise - """ - return self.resource_manager.is_resource_sufficient(input_token_num) - - def all_tasks_finished(self): - """ - judge if all tasks are finished - - Returns: - return: True if all finished, False otherwise - """ - return np.sum(self.resource_manager.stop_flags) == len(self.resource_manager.stop_flags) - - def available_batch(self): - """ - available batch size of the engine - - Returns: - return: available batch size - """ - return self.resource_manager.available_batch() - - def available_block_num(self): - """ - available block number of the engine - - Returns: - return: available block number - """ - return self.resource_manager.availabel_block_num() - - def _set_warmup_token_processor(self): - """ - set token_processor for warmup - """ - self.token_processor_backup = self.token_processor - self.token_processor = WarmUpTokenProcessor(self.cfg) - self.token_processor.set_resource_manager(self.resource_manager) - self.token_processor.tasks_queue = self.tasks_queue - - # start TokenProcessor thread - self.token_processor.run() - - def _del_warmup_token_processor(self): - """ - delete token_processor for warmup - """ - self.token_processor.stop() - del self.token_processor - - # reset token_processor - self.token_processor = self.token_processor_backup - del self.token_processor_backup - - def _infer_processes_ready(self): - """ - judge if all infer processes are ready - - Returns: - return: True if all ready, False otherwise - """ - if np.sum(self.flag_ready_array) == self.cfg.mp_num: - return True - return False - - def _clear_engine_flags(self): - """ - clear engine flags - """ - try: - self.shm_flag_ready.close() - self.shm_flag_ready.unlink() - self.shm_flag_has_block_step.close() - self.shm_flag_has_block_step.unlink() - except: - pass - - def _init_engine_flags(self): - """ - Initialize shared memory to indicate engine status - """ - flag_array = np.zeros([self.cfg.mp_num], dtype=np.int32) - try: - tmp = shared_memory.SharedMemory( - create=False, size=flag_array.nbytes, name=self.cfg.get_unique_name("shm_flag_infer_ready") - ) - tmp.close() - tmp.unlink() - except: - pass - self.shm_flag_ready = shared_memory.SharedMemory( - create=True, size=flag_array.nbytes, name=self.cfg.get_unique_name("shm_flag_infer_ready") - ) - self.flag_ready_array = np.ndarray( - flag_array.shape, dtype=flag_array.dtype, buffer=self.shm_flag_ready.buf - ) - self.flag_ready_array[:] = 0 - - # broadcast flag for engine - broadcast_flag_array = np.zeros([1], dtype=np.int32) - try: - tmp = shared_memory.SharedMemory( - create=False, - size=broadcast_flag_array.nbytes, - name=self.cfg.get_unique_name("shm_pd_infer_flag_broadcast"), - ) - tmp.close() - tmp.unlink() - except: - pass - self.shm_flag_broadcast = shared_memory.SharedMemory( - create=True, size=broadcast_flag_array.nbytes, name=self.cfg.get_unique_name("shm_pd_infer_flag_broadcast") - ) - self.flag_broadcast_array = np.ndarray( - broadcast_flag_array.shape, - dtype=broadcast_flag_array.dtype, - buffer=self.shm_flag_broadcast.buf, - ) - self.flag_broadcast_array[0] = 0 - - has_block_step_flag_array = np.zeros([1], dtype=np.int32) - try: - tmp = shared_memory.SharedMemory( - create=False, - size=has_block_step_flag_array.nbytes, - name=self.cfg.get_unique_name("shm_flag_has_block_step")) - tmp.close() - tmp.unlink() - except: - pass - self.shm_flag_has_block_step = shared_memory.SharedMemory( - create=True, - size=has_block_step_flag_array.nbytes, - name=self.cfg.get_unique_name("shm_flag_has_block_step")) - self.flag_has_block_step_array = np.ndarray( - has_block_step_flag_array.shape, - dtype=has_block_step_flag_array.dtype, - buffer=self.shm_flag_has_block_step.buf) - self.flag_has_block_step_array[:] = 0 - - def _exit_sub_services(self): - """ - exit sub services - """ - if hasattr(self, "queue_service") and self.queue_service is not None: - self.queue_service.terminate() - self.queue_service.join() - if hasattr(self, "infer_proc") and self.infer_proc is not None: - os.killpg(self.infer_proc.pid, signal.SIGTERM) - - def _start_tasks_queue_service(self): - """ - start tasks queue service - - Returns: - p: process handle - """ - p = multiprocessing.Process(target=launch_queue_service, args=(self.cfg.infer_port, self.cfg.mp_num)) - p.start() - time.sleep(0.3) - if p.is_alive(): - model_server_logger.info("start tasks queue service successfully") - else: - error_msg = "Failed to start tasks queue service, please check " \ - "the log/task_queue_manager.log for details" - model_server_logger.info(error_msg) - raise Exception(error_msg) - return p - - def _start_gpu_infer_service(self): - """ - start gpu infer service - - Returns: - p: process handle - """ - current_file_path = os.path.abspath(__file__) - current_dir_path = os.path.split(current_file_path)[0] - pd_cmd = "python3 -m paddle.distributed.launch " - py_script = os.path.join(current_dir_path, "infer.py") - - arguments = (f" --devices {self.cfg.device_ids} {py_script} --model_dir {self.cfg.model_dir}" - f" --max_batch_size {self.cfg.max_batch_size} --max_seq_len {self.cfg.max_seq_len}" - f" --max_dec_len {self.cfg.max_dec_len}" - f" --max_block_num {self.cfg.total_block_num} --block_size {self.cfg.block_size}" - f" --use_cache_kv_int8 {self.cfg.use_cache_kv_int8}" - f" --enc_dec_block_num {self.cfg.enc_dec_block_num}" - f" --block_ratio {self.cfg.block_ratio} --dtype {self.cfg.dtype}") - pd_cmd = pd_cmd + arguments + " >log/launch_infer.log 2>&1" - model_server_logger.info("Launch infer service command: {}".format(pd_cmd)) - p = subprocess.Popen( - pd_cmd, - shell=True, - preexec_fn=os.setsid, - ) - return p - - def _start_infer_service(self): - """ - start infer service - """ - return self._start_gpu_infer_service() diff --git a/services/llm/server/server/engine/infer.py b/services/llm/server/server/engine/infer.py deleted file mode 100644 index dd889cf5418..00000000000 --- a/services/llm/server/server/engine/infer.py +++ /dev/null @@ -1,697 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import copy -import json -import os -import sys -import time -from concurrent.futures import ThreadPoolExecutor -from multiprocessing import shared_memory - -import numpy as np -import paddle -import paddle.distributed as dist -import paddle.distributed.fleet as fleet -from paddlenlp.trl.llm_utils import get_rotary_position_embedding -from paddlenlp_ops import step_paddle -from server.data.processor import DataProcessor -from server.engine.config import Config -from paddlenlp.experimental.transformers import InferenceWithReferenceProposer -from server.utils import get_logger -from task_queue_manager import TaskQueueManager - -File_Path = os.path.realpath(sys.argv[0]) -Dir_Path = os.path.dirname(File_Path) -logger = get_logger("infer_server", "infer.log") - - -class ModelRunner: - def __init__(self, args): - self.args = args - - # 2**63 - 1 - self.MAX_INFER_SEED = 9223372036854775806 - - self.config = Config() - self.model_cfg = self.config.get_model_config() - self.speculate_config = self.config.get_speculate_config() - self.is_speculate_decoding = self.speculate_config.speculate_method != "None" - self.format_print_configuration() - - self.args.num_layers = self.get_value(self.model_cfg, ["num_hidden_layers", "num_layers"]) - self.args.num_attention_heads = self.get_value(self.model_cfg, ["num_attention_heads", "n_head"]) - self.args.hidden_size = self.model_cfg["hidden_size"] - - self.reduce_dialogue_repetition = int(os.environ.get("REDUCE_DIALOGUE_REPETITION", 0)) - - self.max_stop_seqs_num = int(os.getenv("MAX_STOP_SEQS_NUM", 5)) - self.stop_seqs_max_len = int(os.getenv("STOP_SEQS_MAX_LEN", 8)) - - self.nranks = dist.get_world_size() - self.init_dist_env() - self.rank = fleet.worker_index() - - self.load_model_init_val() - - self.share_inputs = {} - self.cache_kvs = {} - self.init_inputs() - - self.proposer = None - if self.is_speculate_decoding: - logger.info(f'Using speculate decoding, method: {self.speculate_config.speculate_method}.') - if self.speculate_config.speculate_method == "inference_with_reference": - self.proposer = InferenceWithReferenceProposer( - self.speculate_config.speculate_max_draft_token_num, - self.speculate_config.speculate_max_ngram_size, - self.args.max_batch_size, - self.args.max_seq_len) - else: - logger.error( - f'Unsupported speculate method: {self.speculate_config.speculate_method}, disabling speculate decoding.' - ) - self.is_speculate_decoding = False - - self.infer_queue = TaskQueueManager(rank=self.rank, mp_num=self.nranks, port=self.config.infer_port) - - model_rank_path = os.path.join(self.args.model_dir, f"rank_{self.rank}") - if not os.path.exists(model_rank_path): - model_rank_path = self.args.model_dir - - self.infer_engine = InferenceEngine(model_dir=model_rank_path, - share_inputs=self.share_inputs, - cache_kvs=self.cache_kvs, - config=self.config, - mp_degree=self.nranks - ) - - def read_model_config(self): - """ - load model config file from json file - - Returns: - model_config_json: dict, model config file - """ - model_config_json = json.load(open(self.config_file, 'r', encoding='utf-8')) - return model_config_json - - def get_value(self, cfg, names): - """ - get value from config file by key names - """ - if not isinstance(names, list): - names = [names] - for name in names: - if name in cfg: - return cfg[name] - break - raise Exception( - "Cannot find any one of key in {} in configuration file.".format( - names)) - - def format_print_configuration(self): - """ - print model config - """ - logger.info("=============== Model Information ==============") - for k, v in self.model_cfg.items(): - logger.info("{:<20}:{:<6}{}".format(k, "", v)) - logger.info("=============== Service Configuration ===============") - for k, v in vars(self.args).items(): - logger.info("{:<20}:{:<6}{}".format(k, "", v)) - logger.info("=====================================================\n") - - def load_model_init_val(self): - """ - initialize model config from config file - """ - self.top_p = self.model_cfg.get("top_p", 0.0) - self.temperature = self.model_cfg.get("temperature", 1.0) - self.rope_theta = self.model_cfg.get('rope_theta', 10000.0) - self.rope_scaling = self.model_cfg.get('rope_scaling', None) - self.penalty_score = self.model_cfg.get('penalty_score', 1.0) - self.frequency_score = self.model_cfg.get('frequency_score', 0.0) - self.presence_score = self.model_cfg.get('presence_score', 0.0) - self.min_length = self.model_cfg.get('min_length', 1) - self.max_length = self.model_cfg.get('max_length', 1024) - - data_processor = DataProcessor() - # reserve an eos token for request - self.eos_tokens_lens = data_processor.get_eos_tokens_lens() + 1 - self.pad_token_id = data_processor.get_pad_id() - - def init_dist_env(self, seed=20): - """ - init distributed env - """ - strategy = fleet.DistributedStrategy() - - strategy.hybrid_configs = { - "dp_degree": 1, - "mp_degree": self.nranks, - "pp_degree": 1, - "sharding_degree": 1, - } - - # Set control in tensor parallel - strategy.tensor_parallel_configs = {"tensor_init_seed": seed} - fleet.init(is_collective=True, strategy=strategy) - - def init_inputs(self): - # init all inputs - if "num_key_value_heads" in self.model_cfg and \ - self.model_cfg["num_key_value_heads"] is not None and \ - int(self.model_cfg["num_key_value_heads"]) > 0: - kv_num_head = int(self.model_cfg["num_key_value_heads"]) // self.nranks - else: - kv_num_head = self.args.num_attention_heads // self.nranks - - for i in range(self.args.num_layers): - if not self.args.use_cache_kv_int8: - cache_type = self.args.dtype - else: - cache_type = "uint8" - - self.cache_kvs["key_caches_{}".format(i)] = paddle.full(shape=[ - self.args.max_block_num, kv_num_head, - self.args.block_size, self.args.hidden_size // self.args.num_attention_heads - ], fill_value=0, dtype=cache_type) - self.cache_kvs["value_caches_{}".format(i)] = paddle.full(shape=[ - self.args.max_block_num, kv_num_head, - self.args.block_size, self.args.hidden_size // self.args.num_attention_heads - ], fill_value=0, dtype=cache_type) - - pre_max_block_num = (self.args.max_seq_len + self.args.block_size - 1) // self.args.block_size + self.args.enc_dec_block_num - self.share_inputs["block_tables"] = paddle.full( - shape=[self.args.max_batch_size, pre_max_block_num], fill_value=-1, dtype="int32") - - self.share_inputs['pre_ids'] = paddle.to_tensor( - np.full((self.args.max_batch_size, self.args.max_dec_len), -1, dtype='int64')) - - tmp_position_ids = paddle.arange(self.args.max_seq_len).reshape((1, -1)) - self.share_inputs['rope_emb'] = get_rotary_position_embedding(tmp_position_ids, - self.args.hidden_size // self.args.num_attention_heads, - self.rope_theta, self.rope_scaling) - self.share_inputs['input_ids'] = paddle.full( - shape=[self.args.max_batch_size, self.args.max_seq_len], - fill_value=self.pad_token_id, dtype='int64') - self.share_inputs['top_p'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=self.top_p, dtype="float32") - self.share_inputs['temperature'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=self.temperature, dtype="float32") - self.share_inputs['eos_token_id'] = paddle.to_tensor( - np.zeros((self.eos_tokens_lens, 1)).reshape(-1, 1).astype("int64")) - self.share_inputs['penalty_score'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=self.penalty_score, dtype="float32") - self.share_inputs['frequency_score'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=self.frequency_score, dtype="float32") - self.share_inputs['presence_score'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=self.presence_score, dtype="float32") - self.share_inputs['seq_lens_this_time'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32") - self.share_inputs['seq_lens_encoder'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32") - self.share_inputs['step_seq_lens_encoder'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32") - self.share_inputs['seq_lens_decoder'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32") - self.share_inputs['step_idx'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int64") - self.share_inputs['min_length'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=self.min_length, dtype="int64") - self.share_inputs['max_length'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=self.max_length, dtype="int64") - self.share_inputs['not_need_stop'] = paddle.full( - shape=[1], fill_value=False, dtype="bool") - self.share_inputs['stop_flags'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=True, dtype="bool") - self.share_inputs['stop_nums'] = paddle.full( - shape=[1], fill_value=self.args.max_batch_size, dtype="int64") - self.share_inputs['bad_tokens'] = paddle.full( - shape=[1], fill_value=-1, dtype="int64") - self.share_inputs['next_tokens'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=-1, dtype="int64") - self.share_inputs['is_block_step'] = paddle.full( - shape=[self.args.max_batch_size], fill_value=False, dtype="bool") - self.share_inputs['encoder_block_lens'] = paddle.full( - shape=[self.args.max_batch_size], fill_value=0, dtype="int32") - self.share_inputs['step_block_list'] = paddle.full( - shape=[self.args.max_batch_size], fill_value=-1, dtype="int32") - self.share_inputs['step_lens'] = paddle.full(shape=[1], fill_value=0, dtype="int32") - self.share_inputs['recover_block_list'] = paddle.full( - shape=[self.args.max_batch_size], fill_value=-1, dtype="int32") - self.share_inputs['recover_lens'] = paddle.full( - shape=[1], fill_value=0, dtype="int32") - self.share_inputs['need_block_list'] = paddle.full( - shape=[self.args.max_batch_size], fill_value=-1, dtype="int32") - self.share_inputs['need_block_len'] = paddle.full( - shape=[1], fill_value=0, dtype="int32") - self.share_inputs['used_list_len'] = paddle.full( - shape=[self.args.max_batch_size], fill_value=0, dtype="int32") - self.share_inputs['infer_seed'] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int64") - free_list = list(range(int(self.args.max_block_num * self.args.block_ratio))) - self.free_list_len = len(free_list) - self.share_inputs['free_list'] = paddle.to_tensor(free_list, dtype="int32") - self.share_inputs['free_list_len'] = paddle.full( - shape=[1], fill_value=self.free_list_len, dtype="int32") - - self.share_inputs['stop_seqs_len'] = paddle.full( - shape=[self.args.max_batch_size, self.max_stop_seqs_num], - fill_value=0, - dtype="int32") - self.share_inputs['stop_seqs'] = paddle.full( - shape=[self.args.max_batch_size, self.max_stop_seqs_num, self.stop_seqs_max_len], - fill_value=-1, - dtype="int64") - - if self.reduce_dialogue_repetition: - self.share_inputs["first_token_ids"] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=-1, dtype="int64") - self.share_inputs["ori_seq_lens_encoder"] = paddle.full( - shape=[self.args.max_batch_size, 1], fill_value=0, dtype="int32") - # speculate decoding input - if self.is_speculate_decoding: - self.share_inputs["accept_tokens"] = paddle.full( - shape=[self.args.max_batch_size, self.speculate_config.speculate_max_draft_token_num + 1], fill_value=0, dtype="int64" - ) - self.share_inputs["accept_num"] = paddle.full(shape=[self.args.max_batch_size], fill_value=0, dtype="int32") - self.share_inputs["draft_tokens"] = paddle.full( - shape=[self.args.max_batch_size, self.speculate_config.speculate_max_draft_token_num + 1], fill_value=0, dtype="int64" - ) - self.share_inputs["actual_draft_token_num"] = paddle.full( - shape=[self.args.max_batch_size], fill_value=self.speculate_config.speculate_max_draft_token_num, dtype="int32" - ) - - def dy_input_preprocess(self, tasks): - """ - dynamic insertion - """ - for i in range(len(tasks)): - task = tasks[i] - idx = task['idx'] - length = len(task['input_ids']) - self.share_inputs['input_ids'][idx:idx + 1, :length] = np.array(task['input_ids']) - if len(task['eos_token_ids']) < self.eos_tokens_lens: - task['eos_token_ids'].append(task['eos_token_ids'][0]) - self.share_inputs['eos_token_id'][:] = np.array(task['eos_token_ids'], dtype="int64").reshape(-1, 1) - self.share_inputs['pre_ids'][idx:idx + 1] = -1 - self.share_inputs['top_p'][idx:idx + 1] = task.get('topp', 0.7) - self.share_inputs['temperature'][idx:idx + 1] = task.get('temperature', 0.95) - self.share_inputs['penalty_score'][idx:idx + 1] = task.get('penalty_score', 1.0) - self.share_inputs['frequency_score'][idx:idx + 1] = task.get('frequency_score', 0.0) - self.share_inputs['presence_score'][idx:idx + 1] = task.get('presence_score', 0.0) - self.share_inputs['seq_lens_this_time'][idx:idx + 1] = length - self.share_inputs['step_seq_lens_encoder'][idx:idx + 1] = length - self.share_inputs['seq_lens_encoder'][idx:idx + 1] = length - self.share_inputs['seq_lens_decoder'][idx:idx + 1] = 0 - self.share_inputs['step_idx'][idx:idx + 1] = 0 - self.share_inputs['min_length'][idx:idx + 1] = task.get('min_dec_len', 1) - if "max_dec_len" in task: - max_dec_len = task['max_dec_len'] - elif "seq_len" in task: - max_dec_len = task['seq_len'] - else: - max_dec_len = self.args.max_dec_len - self.share_inputs['max_length'][idx:idx + 1] = max_dec_len - self.share_inputs['stop_flags'][idx:idx + 1] = False - - if self.reduce_dialogue_repetition: - self.share_inputs['first_token_ids'][idx:idx + 1] = self.share_inputs['input_ids'][idx:idx + 1, :1] - self.share_inputs["ori_seq_lens_encoder"][idx:idx + 1] = length - - if "infer_seed" in task: - self.share_inputs['infer_seed'][idx:idx + 1] = task['infer_seed'] - - encoder_block_num = len(task['block_tables']) - self.share_inputs['encoder_block_lens'][idx:idx + 1] = encoder_block_num - self.share_inputs["block_tables"][idx:idx + 1, :] = -1 - self.share_inputs["block_tables"][idx:idx + 1, :encoder_block_num] = np.array( - task['block_tables'], dtype="int32") - - self.share_inputs['stop_seqs_len'][idx:idx + 1, :] = 0 - self.share_inputs['stop_seqs'][idx:idx + 1, :, :] = -1 - - if "stop_seqs_len" in task and "stop_seqs" in task: - raw_stop_seqs = task["stop_seqs"][:self.max_stop_seqs_num] - raw_stop_seqs_len = task["stop_seqs_len"][:self.max_stop_seqs_num] - stop_seqs_num = len(raw_stop_seqs_len) - if stop_seqs_num > 0 and len(raw_stop_seqs) > 0: - truncated_seqs = [] - truncated_lens = [] - for s_len, s_seq in zip(raw_stop_seqs_len, raw_stop_seqs): - seq_trunc = s_seq[:self.stop_seqs_max_len] - truncated_seqs.append(seq_trunc) - truncated_lens.append(min(s_len, self.stop_seqs_max_len)) - - padded_lens = list(truncated_lens) - for _ in range(len(padded_lens), self.max_stop_seqs_num): - padded_lens.append(0) - self.share_inputs['stop_seqs_len'][idx:idx + 1, :] = np.array( - padded_lens, dtype="int32") - - max_seq_len = max(len(s) for s in truncated_seqs) if truncated_seqs else 0 - if max_seq_len > 0: - padded_seqs = [ - list(s) + [-1] * (max_seq_len - len(s)) for s in truncated_seqs - ] - self.share_inputs['stop_seqs'][idx:idx + 1, :stop_seqs_num, :max_seq_len] = np.array( - padded_seqs, dtype="int64") - - if self.is_speculate_decoding: - self.share_inputs["draft_tokens"][idx:idx + 1] = np.zeros([self.speculate_config.speculate_max_draft_token_num + 1]) - self.share_inputs["actual_draft_token_num"][idx:idx + 1] = np.array([self.speculate_config.speculate_max_draft_token_num]) - - def step_cuda(self, seq_lens_this_time): - """ - step cuda - """ - # whether speculate decoding - if self.is_speculate_decoding: - speculate_step_token_num = self.speculate_config.speculate_max_draft_token_num + 1 - else: - speculate_step_token_num = 0 - - step_paddle(self.share_inputs['stop_flags'], seq_lens_this_time, - self.share_inputs['step_seq_lens_encoder'], - self.share_inputs['seq_lens_encoder'], - self.share_inputs['seq_lens_decoder'], self.share_inputs["block_tables"], - self.share_inputs['encoder_block_lens'], - self.share_inputs["is_block_step"], self.share_inputs['step_block_list'], - self.share_inputs['step_lens'], self.share_inputs['recover_block_list'], - self.share_inputs['recover_lens'], self.share_inputs['need_block_list'], - self.share_inputs['need_block_len'], self.share_inputs['used_list_len'], - self.share_inputs['free_list'], self.share_inputs['free_list_len'], - self.share_inputs['input_ids'], self.share_inputs['pre_ids'], - self.share_inputs['step_idx'], self.share_inputs['next_tokens'], - self.args.block_size, self.args.enc_dec_block_num, self.args.first_token_id, - speculate_step_token_num) - - def initialize_engine_ready_check_flag(self): - """ - initialize engine ready flag in shared memory - - Returns: - shm_engine_ready_check_flag: engine ready flag - engine_ready_check_flag_array: engine ready flag array - """ - engine_ready_check_flag = np.zeros([1], dtype=np.int32) - shm_engine_ready_check_flag = shared_memory.SharedMemory( - name=self.config.get_unique_name("engine_ready_check_flag")) - engine_ready_check_flag_array = np.ndarray(engine_ready_check_flag.shape, - dtype=engine_ready_check_flag.dtype, - buffer=shm_engine_ready_check_flag.buf) - return shm_engine_ready_check_flag, engine_ready_check_flag_array - - def initialize_engine_live_flag(self): - """ - initialize infer live flag in shared memory - - Returns: - infer_live_flag_shm: infer live flag - """ - infer_live_flag_shm = shared_memory.SharedMemory(create=True, - size=1, - name=self.config.get_unique_name("shm_flag_infer_{}_live".format(self.rank))) - return infer_live_flag_shm - - def initialize_engine_healthy_recorded_time_flag(self): - """ - initialize engine healthy recorded time flag in shared memory - - Returns: - shm_engine_healthy_recorded_time: engine healthy recorded time flag - """ - engine_healthy_recorded_time = np.zeros([1], dtype=float) - shm_engine_healthy_recorded_time = shared_memory.SharedMemory( - name=self.config.get_unique_name("engine_healthy_recorded_time")) - engine_healthy_recorded_time_array = np.ndarray(engine_healthy_recorded_time.shape, - dtype=engine_healthy_recorded_time.dtype, - buffer=shm_engine_healthy_recorded_time.buf) - return shm_engine_healthy_recorded_time, engine_healthy_recorded_time_array - - def run(self): - """ - run infer - """ - flag_array = np.zeros([1], dtype=np.int32) - shm_flag_broadcast = shared_memory.SharedMemory( - name=self.config.get_unique_name("shm_pd_infer_flag_broadcast")) - flag_broadcast_array = np.ndarray(flag_array.shape, - dtype=flag_array.dtype, - buffer=shm_flag_broadcast.buf) - - flag_array = np.zeros([self.nranks], dtype=np.int32) - shm_flag_ready = shared_memory.SharedMemory(name=self.config.get_unique_name("shm_flag_infer_ready")) - flag_ready_array = np.ndarray(flag_array.shape, - dtype=flag_array.dtype, - buffer=shm_flag_ready.buf) - flag_ready_array[self.rank] = 1 - - flag_array = np.zeros([1], dtype=np.int32) - shm_flag_has_block_step = shared_memory.SharedMemory(name=self.config.get_unique_name("shm_flag_has_block_step")) - flag_has_block_step_array = np.ndarray(flag_array.shape, - dtype=flag_array.dtype, - buffer=shm_flag_has_block_step.buf) - - use_custom_health_checker = self.config.use_custom_health_checker - if use_custom_health_checker: - shm_engine_ready_check_flag_array, engine_ready_check_flag_array = self.initialize_engine_ready_check_flag() - engine_ready_check_flag_array[0] = 1 - shm_engine_healthy_recorded_time_array, engine_healthy_recorded_time_array = self.initialize_engine_healthy_recorded_time_flag() - engine_healthy_recorded_time_array[0] = time.time() - infer_live_flag_shm = self.initialize_engine_live_flag() - infer_seed_increment = paddle.full(shape=[self.args.max_batch_size, 1], - fill_value=4, - dtype="int64") - thread_executor = ThreadPoolExecutor(max_workers=1) - seq_lens_this_time = None - real_bsz = None - - while True: - if use_custom_health_checker: - engine_healthy_recorded_time_array[0] = time.time() - - if self.rank == 0: - if not self.infer_queue.empty(): - flag_broadcast_array[0] = 1 - - if self.nranks > 1: - paddle.distributed.barrier() - - if flag_broadcast_array[0] == 1: - logger.info(f'rank: {self.rank} start to get') - if seq_lens_this_time is not None: - self.share_inputs["seq_lens_this_time"][:real_bsz] = seq_lens_this_time - - tasks, read_finish = self.infer_queue.get() - if read_finish: - flag_broadcast_array[0] = 0 - - req_dicts = [] - for req_dict, bsz in tasks: - real_bsz = int(bsz) - req_dicts.extend(req_dict) - logger.info( - f'rank: {self.rank}, real_bsz: {real_bsz}, query_num: {len(req_dicts)}' - ) - - self.dy_input_preprocess(req_dicts) - seq_lens_this_time = copy.deepcopy( - self.share_inputs['seq_lens_this_time'][:real_bsz]) - self.infer_engine.seq_lens_handle.share_external_data( - seq_lens_this_time) - self.share_inputs['not_need_stop'][0] = True - - if not self.share_inputs['not_need_stop']: - if self.nranks > 1: - paddle.distributed.barrier() - - time.sleep(0.001) - continue - - if self.proposer is not None: - self.proposer.run( - self.share_inputs, - real_batch_size=seq_lens_this_time.shape[0], - seq_lens_this_time=seq_lens_this_time, - ) - - self.infer_engine.predictor.run() - self.share_inputs['infer_seed'].add_(infer_seed_increment) - self.share_inputs['infer_seed'][:] %= self.MAX_INFER_SEED - if self.free_list_len > 0: - self.step_cuda(seq_lens_this_time) - - -class InferenceEngine(object): - """ - Model Parallel Inference Engine - - Args: - model_dir (string): root directory of inference model - mp_degree (int): model parallel size - """ - def __init__(self, model_dir, share_inputs, cache_kvs, config, mp_degree=1): - self.config = config - self.model_dir = model_dir - self.mp_degree = mp_degree - - self.share_inputs = share_inputs - self.cache_kvs = cache_kvs - - if mp_degree == 1: - self.nranks = 1 - self.rank = 0 - else: - self.nranks = fleet.worker_num() - self.rank = fleet.worker_index() - - self._init_predictor() - self.share_data() - - def _init_predictor(self): - """ - predictor init - """ - device_id = self.rank % 8 - self.model_file = os.path.join(self.model_dir, f"model.pdmodel") - self.param_file = os.path.join(self.model_dir, f"model.pdiparams") - config = paddle.inference.Config(self.model_file, self.param_file) - - config.switch_ir_optim(False) - config.enable_use_cuda(100, device_id) - - pir_flag = int(os.environ.get("FLAGS_enable_pir_api", 0)) - if pir_flag == 1: - config.enable_new_executor() - config.enable_new_ir() - - # distributed config - if self.mp_degree > 1: - trainer_endpoints = fleet.worker_endpoints() - current_endpoint = trainer_endpoints[self.rank] - dist_config = config.dist_config() - dist_config.set_ranks(self.nranks, self.rank) - dist_config.set_endpoints(trainer_endpoints, current_endpoint) - dist_config.enable_dist_model(True) - if self.config.distributed_config_path: - dist_config.set_comm_init_config(self.config.distributed_config_path) - else: - raise Exception("Please set DISTRIBUTED_CONFIG env variable.") - logger.warning( - f"Use default distributed config, please set env DISTRIBUTED_CONFIG" - ) - dist_config.set_comm_init_config( - os.path.join(Dir_Path + "/config", "rank_mapping_mp{}.csv".format(self.nranks))) - - config.set_dist_config(dist_config) - self.predictor = paddle.inference.create_predictor(config) - self.input_names = self.predictor.get_input_names() - self.seq_lens_handle = self.predictor.get_input_handle('seq_lens_this_time') - - def share_data(self): - """ - share data - """ - for name in self.input_names: - if "caches" in name: - input_tensor = self.predictor.get_input_handle(name) - input_tensor.share_external_data(self.cache_kvs[name]) - continue - if "seq_lens_this_time" in name: - continue - input_tensor = self.predictor.get_input_handle(name) - input_tensor.share_external_data(self.share_inputs[name]) - - def predict(self, real_bsz): - """ - predict - """ - seq_lens_this_time = copy.deepcopy( - self.share_inputs['seq_lens_this_time'][:real_bsz]) - self.seq_lens_handle.share_external_data(seq_lens_this_time) - self.share_inputs['not_need_stop'][0] = True - while self.share_inputs['not_need_stop']: - self.predictor.run() - self.share_inputs["seq_lens_this_time"][:real_bsz] = seq_lens_this_time - - -def parse_args(): - """ - parse args from command line - """ - parser = argparse.ArgumentParser("FastDeploy LLM Inference") - parser.add_argument('-m', - '--model_dir', - type=str, - default='./output', - help='model dir') - parser.add_argument('-mp', - '--mp_degree', - type=int, - default=1, - help='mp degree') - parser.add_argument('-mbs', - '--max_batch_size', - type=int, - default=34, - help='max batch size') - parser.add_argument('--max_block_num', type=int, default=2000) - parser.add_argument("--block_size", type=int, default=128) - parser.add_argument('--max_seq_len', - type=int, - default=3072, - help='max_seq_len') - parser.add_argument('--max_dec_len', - type=int, - default=1024, - help='max_dec_len') - parser.add_argument('--use_cache_kv_int8', - type=int, - default=0, - help='use cache kv int8') - parser.add_argument('--dtype', - type=str, - default="bfloat16", - help='input dtype') - parser.add_argument('--enc_dec_block_num', - type=int, - default=1, - help="encoder's decoder num") - parser.add_argument('--block_ratio', - type=float, - default=0.7, - help="block ratio") - parser.add_argument('--first_token_id', - type=int, - default=1, - help="first token id") - args = parser.parse_args() - return args - - -def main(): - """ - start model runner - """ - args = parse_args() - model_runner = ModelRunner(args) - model_runner.run() - - -if __name__ == "__main__": - main() diff --git a/services/llm/server/server/engine/resource_manager.py b/services/llm/server/server/engine/resource_manager.py deleted file mode 100644 index 148e610b287..00000000000 --- a/services/llm/server/server/engine/resource_manager.py +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import os -import random -import threading -import time - -import numpy as np -from server.utils import model_server_logger - - -class ResourceManager(object): - """ - record and allocate resources for the engine - """ - def __init__(self, cfg): - self.cfg = cfg - self.stop_flags = [True] * cfg.max_batch_size - self.free_list = list(range(cfg.max_block_num - 1, -1, -1)) - self.tasks_list = [None] * self.cfg.max_batch_size - # current batch status of the engine - self.real_bsz = 0 - model_server_logger.info(f"{self.info()}") - - def get_required_block_number(self, input_token_num): - """ - Calculate Block resources are needed - - Args: - input_token_num (int): input token number - - Returns: - int: block number - """ - block_num = (input_token_num + self.cfg.block_size - 1 + self.cfg.dec_token_num) // self.cfg.block_size - return block_num - - def get_encoder_block_number(self, input_token_num): - """ - get the number of blocks for the encoder - - Args: - input_token_num (int): input token number - - Returns: - int: encoder block number - """ - enc_block_num = (input_token_num + self.cfg.block_size - 1) // self.cfg.block_size - return enc_block_num - - def get_decoder_block_number(self): - """ - get the number of blocks for the decoder - - Returns: - int: decoder block number - """ - return (self.cfg.dec_token_num + self.cfg.block_size - 1) // self.cfg.block_size - - def total_block_number(self): - """ - the number of pre allocated blocks at service startup - - Returns: - int: total block number - """ - return self.cfg.max_block_num - - def _get_block_tables(self, input_token_num, required_type="all"): - """ - allocate memory resources - - Args: - input_token_num (int): input token number - required_type (str): required type - - Returns: - list: block list - """ - if required_type == "all": - block_num = self.get_required_block_number(input_token_num) - elif required_type == "encoder": - block_num = self.get_encoder_block_number(input_token_num) - elif required_type == "decoder": - block_num = self.get_decoder_block_number() - else: - raise ValueError('unknown required type') - block_num = min(block_num, self.cfg.max_query_block_num) - block_list = list() - if block_num > len(self.free_list): - model_server_logger.error("block_num:{0} > free_list len:{1}".format(block_num, len(self.free_list))) - return block_list - for _ in range(block_num): - used_block_id = self.free_list.pop() - block_list.append(used_block_id) - model_server_logger.info(f"dispatch {len(block_list)} blocks.") - return block_list - - def _recycle_block_tables(self, block_tables): - """ - Recycling memory resource blocks - - Args: - block_tables (list): block list - """ - ori_number = len(self.free_list) - self.free_list.extend(block_tables) - cur_number = len(self.free_list) - model_server_logger.info(f"recycle {cur_number - ori_number} blocks.") - - def available_batch(self): - """ - available batch size for engine - - Returns: - int: available batch size - """ - return np.sum(self.stop_flags) - - def availabel_block_num(self): - """ - available block size for engine - - Returns: - int: available block size - """ - return len(self.free_list) - - def is_resource_sufficient(self, input_token_num): - """ - check current available resources meet the new requirements - - Args: - input_token_num (int): input token number - - Returns: - bool: whether current available resources meet the new requirements - """ - if self.available_batch() < 1: - return False - block_num = self.get_required_block_number(input_token_num) - if block_num > self.availabel_block_num(): - return False - return True - - def allocate_resources_for_new_tasks(self, tasks): - """ - allocate resources for new tasks - - Args: - tasks (list): task list - - Returns: - list: processed task list - """ - - allocated_position = 0 - processing_task_index = 0 - processed_tasks = list() - while allocated_position < self.cfg.max_batch_size: - if processing_task_index >= len(tasks): - break - - if len(tasks[processing_task_index]["input_ids"]) > self.cfg.max_seq_len: - model_server_logger.error("req_id: {0} input_ids len:{1} > {2}".format( - tasks[ - processing_task_index]["req_id"], len(tasks[ - processing_task_index]["input_ids"]), self.cfg.max_seq_len - )) - processing_task_index += 1 - continue - - can_insert = False - while allocated_position + 1 <= self.cfg.max_batch_size: - if sum(self.stop_flags[allocated_position : allocated_position + 1]) == 1: - can_insert = True - break - allocated_position += 1 - if can_insert: - if self.stop_flags[allocated_position]: - task = copy.deepcopy(tasks[processing_task_index]) - - if not isinstance(task["eos_token_ids"], list): - task["eos_token_ids"] = [task["eos_token_ids"]] - - if "infer_seed" in task and task["infer_seed"]: - task["infer_seed"] = int(task["infer_seed"]) - else: - task["infer_seed"] = random.randint(0, 9223372036854775807) - task["idx"] = allocated_position - task["block_tables"] = self._get_block_tables(len(task["input_ids"])) - if not task["block_tables"]: - model_server_logger.error("req_id: {0} block_tables is empty".format(task["req_id"])) - continue - - processed_tasks.append(task) - self.stop_flags[allocated_position] = False - task["inference_start_time"] = time.time() - task["inference_time_cost"] = -1.0 - task["tokens_all_num"] = int(0) - self.tasks_list[allocated_position] = task - model_server_logger.info(f"allocate req_id: {task['req_id']}, " - f"allocated_position:{allocated_position}, input_ids_length: {len(task['input_ids'])}") - allocated_position += 1 - processing_task_index += 1 - - # batch size when the statistical engine is inferring - for i in range(self.cfg.max_batch_size - 1, -1, -1): - if not self.stop_flags[i]: - self.real_bsz = i + 1 - break - - model_server_logger.info("in num:{0} new task num:{1} real_bsz is:{2}".format( - len(tasks), len(processed_tasks), self.real_bsz)) - model_server_logger.info(f"{self.info()}") - return processed_tasks - - def info(self): - """ - get resource manager info - - Returns: - str: resource manager info - """ - info = f"ResourceManager info, " \ - f"total_block_number: {self.total_block_number()}, total_batch_number: {len(self.stop_flags)}, " \ - f"availabel_block_num: {self.availabel_block_num()}, available_batch: {self.available_batch()}" - return info diff --git a/services/llm/server/server/engine/task_queue_manager.py b/services/llm/server/server/engine/task_queue_manager.py deleted file mode 100644 index 475365d47fb..00000000000 --- a/services/llm/server/server/engine/task_queue_manager.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import threading -import time -from multiprocessing.managers import (AcquirerProxy, BaseManager, ListProxy, - Value, ValueProxy) -from queue import Queue - -from server.utils import get_logger - -logger = get_logger("infer_server", "task_queue_manager.log") - - -class QueueManager(BaseManager): - """ - base class for queue manager - """ - - pass - - -class TaskQueueManager(object): - """ - task queue manager - """ - - def __init__(self, rank=0, mp_num=8, port=56666): - """ - Initialization function, used to perform initialization - operations when creating objects - """ - self.max_get_num = int(os.getenv("ENGINE_MAX_NEED_NUM", 0)) - QueueManager.register('get_list') - QueueManager.register('get_value') - QueueManager.register('get_lock') - QueueManager.register('get_barrier1') - QueueManager.register('get_barrier2') - QueueManager.register('get_queue') - - self.client_manager = QueueManager(address=('127.0.0.1', port), - authkey=b'infer_queue' - ) - self.client_manager.connect() - self.list = self.client_manager.get_list() - self.value = self.client_manager.get_value() - self.lock = self.client_manager.get_lock() - self.barrier1 = self.client_manager.get_barrier1() - self.barrier2 = self.client_manager.get_barrier2() - self.queue = self.client_manager.get_queue() - self.mp_num = mp_num - self.rank = rank - self.position = 1 << rank - self.total_num = (1 << self.mp_num) - 1 - logger.info(f"init task queue manager success, rank: {rank}") - - def empty(self): - """ - check the queue is empty for infer - - Returns: - bool: True if the queue is empty, otherwise False - """ - try: - return len(self.list) == 0 - except Exception as e: - logger.error(f"empty function meets error: {e}") - raise e - - def put(self, item): - """ - put item to queue - - Args: - item (any): the item to put into queue - """ - self.lock.acquire() - if 0 < self.value.get() < self.total_num: - self.lock.release() - while 0 < self.value.get() < self.total_num: - time.sleep(0.001) - logger.info("put item to queue wait finish") - self.lock.acquire() - if self.max_get_num <= 0 and self.value.get() == self.total_num: - self.list[:] = [] - self.value.set(0) - self.list.append(item) - self.lock.release() - logger.info("put item to queue success") - - def get(self): - """ - get item from queue - - Returns: - list: the item from queue - bool: True if the queue is empty, otherwise False - """ - input_list = [] - read_finish = False - self.lock.acquire() - if self.value.get() & self.position == 0 and len(self.list) > 0: - if self.max_get_num > 0: - input_list.extend(self.list[: self.max_get_num]) - else: - input_list.extend(self.list[:]) - set_value = self.value.get() | self.position - logger.info("rank: {0} set_value: {1}".format(self.rank, set_value)) - if set_value >= self.total_num: - if self.max_get_num > 0: - for i in range(self.max_get_num): - self.list.pop(0) - else: - self.list[:] = [] - set_value = 0 - read_finish = True - self.value.set(set_value) - self.lock.release() - return input_list, read_finish - - -def launch_queue_service(port, num_workers): - """ - Start the process communication queue service - - Args: - port (int): the port to listen - num_workers (int): the number of infer process - """ - try: - logger.info(f"start launch queue service, port:{port}") - value = Value("i", 0) - QueueManager.register("get_value", callable=lambda: value, proxytype=ValueProxy) - List = list() - QueueManager.register("get_list", callable=lambda: List, proxytype=ListProxy) - lock = threading.Lock() - QueueManager.register('get_lock', - callable=lambda: lock, - proxytype=AcquirerProxy) - barrier1 = threading.Barrier(num_workers) - QueueManager.register('get_barrier1', callable=lambda: barrier1) - barrier2 = threading.Barrier(num_workers) - QueueManager.register('get_barrier2', callable=lambda: barrier2) - q = Queue() - QueueManager.register("get_queue", callable=lambda: q) - m = QueueManager(address=('127.0.0.1', port), authkey=b'infer_queue') - s = m.get_server() - logger.info("launch queue service success") - s.serve_forever() - logger.info("finish queue service") - except Exception as e: - logger.error(f"launch queue service failed, error_msg: {e}") - raise e diff --git a/services/llm/server/server/engine/token_processor.py b/services/llm/server/server/engine/token_processor.py deleted file mode 100644 index 396145429aa..00000000000 --- a/services/llm/server/server/engine/token_processor.py +++ /dev/null @@ -1,272 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import threading -import time -import traceback -from collections import Counter -from datetime import datetime - -import numpy as np -from paddlenlp_ops import get_output, speculate_get_output -from server.utils import datetime_diff, model_server_logger, monitor_logger -from paddlenlp.utils.env import MAX_DRAFT_TOKENS, SPECULATE_MAX_BSZ - - -class TokenProcessor(object): - """ - get Token/Score from Paddle inference engine - """ - def __init__(self, cfg): - import paddle - paddle.device.set_device("cpu") - self.cfg = cfg - self.resource_manager = None - # record all tokens for each request - self.all_tokens = [[] for _ in range(self.cfg.max_batch_size)] - - self.tokens_counter = Counter() - - self.is_speculate_decoding = self.cfg.get_speculate_config().speculate_method != "None" - if self.is_speculate_decoding: - self.output_tokens = paddle.full(shape=[SPECULATE_MAX_BSZ * MAX_DRAFT_TOKENS + SPECULATE_MAX_BSZ + 2, 1], fill_value=2, dtype="int64") - else: - self.output_tokens = paddle.full(shape=[self.cfg.max_batch_size + 2, 1], fill_value=2, dtype="int64") - self.worker = None - - self.record_time_interval = int(os.getenv("RECORD_TIME_INTERVAL", "600")) - assert self.record_time_interval < 3600, "The RECORD_TIME_INTERVAL cannot exceed 3600." - self.statics_start_time = time.time() - self.number_of_tasks = 0 - self.number_of_input_tokens = 0 - self.number_of_output_tokens = 0 - - def set_resource_manager(self, resource_manager): - """ - set ResourceManager - - Args: - resource_manager (ResourceManager) - """ - assert self.resource_manager is None, "The resource manager is not None, cannot set again." - self.resource_manager = resource_manager - - def run(self): - """ - start thread to get tokens - """ - assert self.resource_manager is not None, "The resource manager is None, cannot run." - if self.worker is not None: - raise Exception("Worker is already running!") - - self.worker = threading.Thread(target=self.process_sampling_results, args=()) - self.worker.daemon = True - self.worker.start() - - def process_sampling_results(self): - """ - read tokens from paddle inference engine and process - """ - while True: - try: - rank_id = 0 - is_blocking = True - if self.is_speculate_decoding: - speculate_get_output(self.output_tokens, rank_id, is_blocking) - else: - get_output(self.output_tokens, rank_id, is_blocking) - - if self.output_tokens[0, 0] == -2: - continue - - self._process_batch_output() - except Exception as e: - model_server_logger.info("while get input_data error: {0} {1}".format(e, str(traceback.format_exc()))) - - def postprocess(self, batch_result, exist_finished_task=False): - """ - single post-processing function - - Args: - batch_result (list): batch results - exist_finished_task (bool): whether there is a finished task - """ - result_dir = "./generate_token_results" - if not os.path.exists(result_dir): - os.makedirs(result_dir) - for result in batch_result: - result_file = os.path.join(result_dir, result["req_id"]) - with open(result_file, "a") as f: - f.write("{}\n".format(result)) - - def _get_single_result(self, i, task_id, token_ids, task): - """ - processing single results - - Args: - i (int): batch index - task_id (str): task id - token_ids (list): token id - task (dict): task information - - Returns: - dict: result - """ - inference_time_cost = time.time() - task["inference_start_time"] - task["inference_time_cost"] = inference_time_cost - task["tokens_all_num"] = len(self.all_tokens[i]) - task["inference_current_step_time"] = datetime.now() - result = { - "req_id": task_id, - "is_end": 0, - "token_ids": token_ids, - "send_idx": self.tokens_counter[task_id], - "inference_time_cost": inference_time_cost, - "infer_seed": task["infer_seed"], - "return_all_tokens": task.get("return_all_tokens", False), - } - - # get benchmark msg - if task.get("benchmark"): - keys = ["preprocess_start_time", "preprocess_end_time", "schedule_start_time", - "inference_start_time", "inference_current_step_time"] - for key in keys: - if key in task: - result[key] = str(task[key]) - - # fill some extra information - result["token_ids"] = [] - for token_id in token_ids: - if token_id in task["eos_token_ids"]: - result["is_end"] = 1 - result["tokens_all_num"] = len(self.all_tokens[i]) + len(result["token_ids"]) + 1 - result["tokens_all_ids"] = self.all_tokens[i] + result["token_ids"] - - info_dict = {} - info_dict["req_id"] = task["req_id"] - info_dict["input_token_num"] = len(task["input_ids"]) - info_dict["output_token_num"] = len(self.all_tokens[i]) + len(result["token_ids"]) - if hasattr(task, "preprocess_start_time") and hasattr(task, "preprocess_end_time"): - info_dict["preprocess_cost_time"] = datetime_diff(task["preprocess_start_time"], - task["preprocess_end_time"]) - if hasattr(task, "preprocess_end_time") and hasattr(task, "schedule_start_time"): - info_dict["cache_waiting_cost_time"] = datetime_diff(task["preprocess_end_time"], - task["schedule_start_time"]) - info_dict["inference_time_cost"] = task["inference_time_cost"] - info_dict["version"] = "OpenSource" - info_dict["timestamp"] = time.time() - monitor_logger.info(f"{info_dict}") - break - else: - result["token_ids"].append(token_id) - - return result - - def _recycle_resources(self, task_id, index, task): - """ - recycle resources - """ - self.resource_manager.stop_flags[index] = True - self.resource_manager.tasks_list[index] = None - self.resource_manager._recycle_block_tables(task["block_tables"]) - if task_id in self.tokens_counter: - del self.tokens_counter[task_id] - self.all_tokens[index] = list() - - def _process_batch_output(self): - """ - batch post-processing function - """ - tokens = self.output_tokens.numpy() - batch = self.output_tokens[1, 0] - if not self.is_speculate_decoding: - tokens = tokens[2:batch + 2] - else: - accept_num = tokens[2:batch + 2] - - batch_result = list() - exist_finished_task = False - for i in range(batch): - if self.resource_manager.stop_flags[i]: - continue - - if not self.is_speculate_decoding: - token_ids = [int(tokens[i, 0])] - else: - token_ids = tokens[2 + SPECULATE_MAX_BSZ + i * MAX_DRAFT_TOKENS: 2 + SPECULATE_MAX_BSZ + i * MAX_DRAFT_TOKENS + accept_num[i, 0], 0].tolist() - - if any(token_id < 0 for token_id in token_ids): - continue - - task = self.resource_manager.tasks_list[i] - - task_id = task["req_id"] - result = self._get_single_result(i, task_id, token_ids, task) - - for token_id in token_ids: - self.tokens_counter[task_id] += 1 - if token_id not in task["eos_token_ids"]: - self.all_tokens[i].append(token_id) - - self.number_of_output_tokens += 1 - if token_id in task["eos_token_ids"]: - self._recycle_resources(task_id, i, task) - model_server_logger.info("req_id: {0} finished".format(task_id)) - model_server_logger.info(f"{self.resource_manager.info()}") - exist_finished_task = True - break - batch_result.append(result) - - self.postprocess(batch_result, exist_finished_task) - - -class WarmUpTokenProcessor(TokenProcessor): - """ - Warmup Processor - """ - def __init__(self, cfg): - super().__init__(cfg) - self._is_running = True - self._is_blocking = True - - def postprocess(self, batch_result, exist_finished_task=False): - pass - - def process_sampling_results(self): - """ - get output from model and process it - """ - while self._is_running: - try: - rank_id = 0 - if self.is_speculate_decoding: - speculate_get_output(self.output_tokens, rank_id, self._is_blocking) - else: - get_output(self.output_tokens, rank_id, self._is_blocking) - - if self.output_tokens[0, 0] == -2: - continue - self._process_batch_output() - except Exception as e: - model_server_logger.info("while get input_data error: {0} {1}".format(e, str(traceback.format_exc()))) - - def stop(self): - """ - stop warm up thread - """ - self._is_running = False - self.worker.join() - model_server_logger.info("warm up thread stop") - del self.worker diff --git a/services/llm/server/server/http_server/__init__.py b/services/llm/server/server/http_server/__init__.py deleted file mode 100644 index fd05a920816..00000000000 --- a/services/llm/server/server/http_server/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/services/llm/server/server/http_server/adapter_openai.py b/services/llm/server/server/http_server/adapter_openai.py deleted file mode 100644 index fe347640358..00000000000 --- a/services/llm/server/server/http_server/adapter_openai.py +++ /dev/null @@ -1,103 +0,0 @@ -import time -import json -import queue - -import numpy as np -from typing import Dict -from datetime import datetime -from functools import partial - -import tritonclient.grpc as grpcclient -from tritonclient import utils as triton_utils -from openai.types.completion_usage import CompletionUsage -from openai.types.completion_choice import CompletionChoice -from openai.types.completion import Completion -from openai.types.chat.chat_completion_chunk import ( - ChoiceDelta, - ChatCompletionChunk, - Choice as ChatCompletionChoice -) - -from server.http_server.api import Req, chat_completion_generator -from server.utils import http_server_logger - - -def format_openai_message_completions(req: Req, result: Dict) -> Completion: - choice_data = CompletionChoice( - index=0, - text=result['token'], - finish_reason=result.get("finish_reason", "stop"), - ) - chunk = Completion( - id=req.req_id, - choices=[choice_data], - model=req.model, - created=int(time.time()), - object="text_completion", - usage=CompletionUsage( - completion_tokens=result["usage"]["completion_tokens"], - prompt_tokens=result["usage"]["prompt_tokens"], - total_tokens=result["usage"]["prompt_tokens"] + result["usage"]["completion_tokens"], - ), - ) - return chunk.model_dump_json(exclude_unset=True) - - -def format_openai_message_chat_completions(req: Req, result: Dict) -> ChatCompletionChunk: - choice_data = ChatCompletionChoice( - index=0, - delta=ChoiceDelta( - content=result['token'], - role="assistant", - ), - finish_reason=result.get("finish_reason", "stop"), - ) - chunk = ChatCompletionChunk( - id=req.req_id, - choices=[choice_data], - model=req.model, - created=int(time.time()), - object="chat.completion.chunk", - usage=CompletionUsage( - completion_tokens=result["usage"]["completion_tokens"], - prompt_tokens=result["usage"]["prompt_tokens"], - total_tokens=result["usage"]["prompt_tokens"] + result["usage"]["completion_tokens"], - ), - ) - return chunk.model_dump_json(exclude_unset=True) - - -def openai_chat_commpletion_generator(infer_grpc_url: str, req: Req, chat_interface: bool) -> Dict: - - def _openai_format_resp(resp_dict): - return f"data: {resp_dict}\n\n" - - for resp in chat_completion_generator(infer_grpc_url, req, yield_json=False): - if resp.get("is_end") == 1: - yield _openai_format_resp("[DONE]") - - if chat_interface: - yield _openai_format_resp(format_openai_message_chat_completions(req, resp)) - else: - yield _openai_format_resp(format_openai_message_completions(req, resp)) - - -def openai_chat_completion_result(infer_grpc_url: str, req: Req, chat_interface: bool): - result = "" - error_resp = None - for resp in chat_completion_generator(infer_grpc_url, req, yield_json=False): - if resp.get("error_msg") or resp.get("error_code"): - error_resp = resp - error_resp["result"] = "" - else: - result += resp.get("token") - usage = resp.get("usage", None) - - if error_resp: - return error_resp - response = {'token': result, 'error_msg': '', 'error_code': 0, 'usage': usage} - - if chat_interface: - return format_openai_message_chat_completions(req, response) - else: - return format_openai_message_completions(req, response) diff --git a/services/llm/server/server/http_server/api.py b/services/llm/server/server/http_server/api.py deleted file mode 100644 index 2e01ae039db..00000000000 --- a/services/llm/server/server/http_server/api.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import queue -import time -import uuid -import shortuuid -from datetime import datetime -from functools import partial -from typing import Dict, List, Optional - -import numpy as np -import tritonclient.grpc as grpcclient -from pydantic import BaseModel, Field -from tritonclient import utils as triton_utils - - -class Req(BaseModel): - req_id: str = Field(default_factory=lambda: str(uuid.uuid4())) - input_ids: Optional[List[int]] = None - text: Optional[str] = None - stop_sequences: Optional[List] = None - messages: Optional[List] = None - max_dec_len: Optional[int] = None - seq_len: Optional[int] = None - min_dec_len: Optional[int] = None - temperature: Optional[float] = None - topp: Optional[float] = None - penalty_score: Optional[float] = None - frequency_score: Optional[float] = None - presence_score: Optional[float] = None - system: Optional[str] = None - return_all_tokens: Optional[bool] = None - eos_token_ids: Optional[List[int]] = None - benchmark: bool = False - return_usage: Optional[bool] = False - stream: bool = False - timeout: int = 300 - model: str = None - - def to_dict_for_infer(self): - """ - Convert the request parameters into a dictionary - - Returns: - dict: request parameters in dict format - """ - req_dict = {} - for key, value in self.dict().items(): - if value is not None: - req_dict[key] = value - return req_dict - - def load_openai_request(self, request_dict: dict): - """ - Convert openai request to Req - official OpenAI API documentation: https://platform.openai.com/docs/api-reference/completions/create - """ - convert_dict = { - "text": "prompt", - "frequency_score": "frequency_penalty", - "max_dec_len": "max_tokens", - "stream": "stream", - "return_all_tokens": "best_of", - "temperature": "temperature", - "topp": "top_p", - "presence_score": "presence_penalty", - "eos_token_ids": "stop", - "req_id": "id", - "model": "model", - "messages": "messages", - } - - self.__setattr__("req_id", f"chatcmpl-{shortuuid.random()}") - for key, value in convert_dict.items(): - if request_dict.get(value, None): - self.__setattr__(key, request_dict.get(value)) - - -def chat_completion_generator(infer_grpc_url: str, req: Req, yield_json: bool) -> Dict: - """ - Chat completion generator based on Triton inference service. - - Args: - infer_grpc_url (str): Triton gRPC URL。 - req (Request): request parameters - yield_json (bool): Whether to return the result in json format - - Returns: - dict: chat completion result. - Normal, return {'token': xxx, 'is_end': xxx, 'send_idx': xxx, ..., 'error_msg': '', 'error_code': 0} - Others, return {'error_msg': xxx, 'error_code': xxx}, error_msg not None, error_code != 0 - """ - class _TritonOutputData: - def __init__(self): - self._completed_requests = queue.Queue() - - def _triton_callback(output_data, result, error): - """Triton callback function""" - if error: - output_data._completed_requests.put(error) - else: - output_data._completed_requests.put(result) - - def _format_resp(resp_dict): - if yield_json: - return json.dumps(resp_dict, ensure_ascii=False) + "\n" - else: - return resp_dict - - timeout = req.timeout - req_id = req.req_id - req_dict = req.to_dict_for_infer() - http_received_time = datetime.now() - - inputs = [grpcclient.InferInput("IN", [1], triton_utils.np_to_triton_dtype(np.object_))] - inputs[0].set_data_from_numpy(np.array([json.dumps([req_dict])], dtype=np.object_)) - outputs = [grpcclient.InferRequestedOutput("OUT")] - output_data = _TritonOutputData() - - with grpcclient.InferenceServerClient(url=infer_grpc_url, verbose=False) as triton_client: - triton_client.start_stream(callback=partial(_triton_callback, output_data)) - - triton_client.async_stream_infer(model_name="model", - inputs=inputs, - request_id=req_dict['req_id'], - outputs=outputs) - while True: - output_item = output_data._completed_requests.get(timeout=timeout) - if type(output_item) == triton_utils.InferenceServerException: - error_msg = f"status is {output_item.status()}, msg is {output_item.message()}" - yield _format_resp({"error_msg": error_msg, "error_code": 500}) - break - else: - result = json.loads(output_item.as_numpy("OUT")[0]) - result = result[0] if isinstance(result, list) else result - result["error_msg"] = result.get("error_msg", "") - result["error_code"] = result.get("error_code", 0) - if req.benchmark: - result["http_received_time"] = str(http_received_time) - yield _format_resp(result) - if (result.get("error_msg") or result.get("error_code")) or result.get("is_end") == 1: - break - - triton_client.stop_stream() - triton_client.close() - -def chat_completion_result(infer_grpc_url: str, req: Req) -> Dict: - """ - Chat completion result with not streaming mode - - Args: - infer_grpc_url (str): Triton gRPC URL - req (Req): request parameters - - Returns: - dict: chat completion result. - Normal, return {'tokens_all': xxx, ..., 'error_msg': '', 'error_code': 0} - Others, return {'error_msg': xxx, 'error_code': xxx}, error_msg not None, error_code != 0 - """ - result = "" - error_resp = None - for resp in chat_completion_generator(infer_grpc_url, req, yield_json=False): - if resp.get("error_msg") or resp.get("error_code"): - error_resp = resp - error_resp["result"] = "" - else: - result += resp.get("token") - usage = resp.get("usage", None) - - if error_resp: - return error_resp - response = {'result': result, 'error_msg': '', 'error_code': 0} - if req.return_usage: - response["usage"] = usage - return response diff --git a/services/llm/server/server/http_server/app.py b/services/llm/server/server/http_server/app.py deleted file mode 100644 index 19351108c06..00000000000 --- a/services/llm/server/server/http_server/app.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os - -import uvicorn -from typing import Dict -from fastapi import FastAPI, Request -from fastapi.responses import StreamingResponse -from server.http_server.api import (Req, chat_completion_generator, - chat_completion_result) -from server.http_server.adapter_openai import ( - openai_chat_commpletion_generator, openai_chat_completion_result -) -from server.utils import http_server_logger - -http_server_logger.info(f"create fastapi app...") -app = FastAPI() - - -@app.post("/v1/chat/completions") -def create_chat_completion(req: Req): - """ - HTTP Server for chat completion - Return: - In Stream: - Normal, return {'token': xxx, 'is_end': xxx, 'send_idx': xxx, ..., 'error_msg': '', 'error_code': 0} - Others, return {'error_msg': xxx, 'error_code': xxx}, error_msg not None, error_code != 0 - Not In Stream: - Normal, return {'tokens_all': xxx, ..., 'error_msg': '', 'error_code': 0} - Others, return {'error_msg': xxx, 'error_code': xxx}, error_msg not None, error_code != 0 - """ - try: - http_server_logger.info(f"receive request: {req.req_id}") - grpc_port = int(os.getenv("GRPC_PORT", 0)) - if grpc_port == 0: - return {"error_msg": f"GRPC_PORT ({grpc_port}) for infer service is invalid", - "error_code": 400} - grpc_url = f"localhost:{grpc_port}" - - if req.stream: - generator = chat_completion_generator(infer_grpc_url=grpc_url, req=req, yield_json=True) - resp = StreamingResponse(generator, media_type="text/event-stream") - else: - resp = chat_completion_result(infer_grpc_url=grpc_url, req=req) - except Exception as e: - resp = {'error_msg': str(e), 'error_code': 501} - finally: - http_server_logger.info(f"finish request: {req.req_id}") - return resp - - -@app.post("/v1/chat/completions/completions") -def openai_v1_completions(request: Dict): - return create_openai_completion(request, chat_interface=False) - - -@app.post("/v1/chat/completions/chat/completions") -def openai_v1_chat_completions(request: Dict): - return create_openai_completion(request, chat_interface=True) - - -def create_openai_completion(request: Dict, chat_interface: bool): - try: - req = Req() - req.load_openai_request(request) - except Exception as e: - return {"error_msg": "request body is not a valid json format", "error_code": 400, "result": ''} - - try: - http_server_logger.info(f"receive request: {req.req_id}") - - grpc_port = int(os.getenv("GRPC_PORT", 0)) - if grpc_port == 0: - return {"error_msg": f"GRPC_PORT ({grpc_port}) for infer service is invalid", - "error_code": 400} - grpc_url = f"localhost:{grpc_port}" - - if req.stream: - generator = openai_chat_commpletion_generator( - infer_grpc_url=grpc_url, - req=req, - chat_interface=chat_interface, - ) - resp = StreamingResponse(generator, media_type="text/event-stream") - else: - resp = openai_chat_completion_result(infer_grpc_url=grpc_url, req=req, chat_interface=chat_interface) - except Exception as e: - resp = {'error_msg': str(e), 'error_code': 501} - finally: - http_server_logger.info(f"finish request: {req.req_id}") - return resp - - -def launch_http_server(port: int, workers: int) -> None: - """ - launch http server - """ - http_server_logger.info(f"launch http server with port: {port}, workers: {workers}") - try: - uvicorn.run(app="server.http_server.app:app", - host='0.0.0.0', - port=port, - workers=workers, - log_level="error") - except Exception as e: - http_server_logger.error(f"launch http server error, {e}") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--port", default=9904, type=int, help="port to the http server") - parser.add_argument("--workers", default=1, type=int, help="set the number of workers for the http service") - args = parser.parse_args() - launch_http_server(port=args.port, workers=args.workers) - - -if __name__ == "__main__": - main() diff --git a/services/llm/server/server/triton_server.py b/services/llm/server/server/triton_server.py deleted file mode 100644 index 02be0b4e8aa..00000000000 --- a/services/llm/server/server/triton_server.py +++ /dev/null @@ -1,490 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import codecs -import json -import multiprocessing -import os -import queue -import subprocess -import sys -import threading -import time -import traceback -from collections import Counter, deque -from datetime import datetime - -import numpy as np -from server.checker import add_default_params, check_basic_params -from server.engine import engine -from server.engine.config import Config -from server.utils import error_logger, model_server_logger - -import server - -try: - import triton_python_backend_utils as pb_utils -except: - model_server_logger.warning( - "TritonPythonModel is only available under triton inference server framework." - ) - -if sys.stdout.encoding is None: - enc = os.environ["LANG"].split(".")[1] - sys.stdout = codecs.getwriter(enc)(sys.stdout) - - -class TritonConfig(Config): - """ - Triton Inference Server config - """ - def __init__(self, base_config): - super().__init__() - for k, v in base_config.__dict__.items(): - setattr(self, k, v) - - -class TritonTokenProcessor(engine.TokenProcessor): - """ - initialize Triton Processor - """ - def __init__(self, cfg, triton_server): - super().__init__(cfg) - self.triton_server = triton_server - self.cached_generated_tokens = queue.Queue() - self.token_buffer = dict() - self.score_buffer = dict() - - self.push_mode_sender_thread = threading.Thread(target=self._push_mode_sender_thread, args=()) - self.push_mode_sender_thread.daemon = True - self.push_mode_sender_thread.start() - - def _push_mode_sender_thread(self): - """ - push mode sender thread - """ - while True: - try: - batch_result = self.cached_generated_tokens.get() - for result in batch_result: - req_id = result["req_id"] - is_end = result.get("is_end", 0) - return_all_tokens = result.get("return_all_tokens", False) - if is_end == 0 and (return_all_tokens or self.cfg.disable_streaming): - continue - if return_all_tokens and "topk_tokens" in result: - del result["topk_tokens"] - result = self.triton_server.data_processor.process_response(result) - if "usage" in result: - result["usage"]["prompt_tokens"] = self.triton_server.task_info[req_id]["prompt_tokens"] - model_server_logger.debug(f"Send result to client under push mode: {result}") - with self.triton_server.thread_lock: - _send_result([result], self.triton_server.response_sender[req_id], is_end) - if is_end == 1: - del self.triton_server.response_sender[req_id] - del self.triton_server.task_info[req_id] - self.triton_server._update_metrics() - except Exception as e: - model_server_logger.error("Unexcepted error happend: {}, {}".format(e, str(traceback.format_exc()))) - - def _cache_special_tokens(self, batch_result): - for i in range(len(batch_result)): - is_end = batch_result[i].get("is_end", 0) - token_ids = batch_result[i]["token_ids"] - if is_end != 1: - if batch_result[i]["req_id"] not in self.token_buffer: - self.token_buffer[batch_result[i]["req_id"]] = list() - self.score_buffer[batch_result[i]["req_id"]] = list() - self.token_buffer[batch_result[i]["req_id"]].extend(token_ids) - self.score_buffer[batch_result[i]["req_id"]].extend(batch_result[i].get("token_scores", [])) - batch_result[i]["token_ids"] = [] - if "token_scores" in batch_result[i]: - batch_result[i]["token_scores"] = [] - else: - if batch_result[i]["req_id"] in self.token_buffer: - batch_result[i]["token_ids"] = self.token_buffer[batch_result[i] - ["req_id"]] + batch_result[i]["token_ids"] - del self.token_buffer[batch_result[i]["req_id"]] - if "token_scores" in batch_result[i]: - batch_result[i]["token_scores"] = self.score_buffer[batch_result[i] - ["req_id"]] + batch_result[i]["token_scores"] - del self.score_buffer[batch_result[i]["req_id"]] - - def postprocess(self, batch_result, exist_finished_task=False): - """ - single postprocess for triton - """ - try: - self._cache_special_tokens(batch_result) - self.cached_generated_tokens.put(batch_result) - except Exception as e: - model_server_logger.info( - "Unexcepted problem happend while process output token: {}, {}" - .format(e, str(traceback.format_exc()))) - - -class TritonServer(object): - """ - Triton Server - """ - - def initialize(self, args): - """ - Triton initialization - """ - # start health checker - use_custom_health_checker = int(os.getenv("USE_CUSTOM_HEALTH_CHECKER", 1)) - # if set USE_CUSTOM_HEALTH_CHECKER=1, use custom health checker, need set --allow-http=false - # else use tritonserver's health checker, need set --http-port=${HTTP_PORT} - if use_custom_health_checker: - http_port = os.getenv("HTTP_PORT") - if http_port is None: - raise Exception("HTTP_PORT must be set") - from server.triton_server_helper import start_health_checker - multiprocessing.Process(target=start_health_checker, args=(int(http_port), )).start() - time.sleep(1) - - model_config = json.loads(args["model_config"]) - using_decoupled = pb_utils.using_decoupled_model_transaction_policy( - model_config) - if not using_decoupled: - raise pb_utils.TritonModelException( - """the model `{}` can generate any number of responses per request, - enable decoupled transaction policy in model configuration to - serve this model""".format(args["model_name"])) - - # add metrics，use METRICS_PORT get server metrics - self.metric_family = pb_utils.MetricFamily( - name="inference_server_metrics", - description="Metrics for monitoring inference server status", - kind=pb_utils.MetricFamily. - GAUGE, - ) - self.metrics = { - "batch_size": - self.metric_family.Metric(labels={"batch_size": "batch_size"}), - "block_num": - self.metric_family.Metric(labels={"block_num": "block_num"}), - "max_batch_size": - self.metric_family.Metric( - labels={"max_batch_size": "max_batch_size"}), - "max_block_num": - self.metric_family.Metric( - labels={"max_block_num": "max_block_num"}), - "available_resource": - self.metric_family.Metric( - labels={"available_resource": "available_resource"}), - } - - # response_sender thread lock - self.thread_lock = threading.Lock() - - base_config = Config() - self.cfg = TritonConfig(base_config) - self.cfg.print(file="log/fastdeploy_init.info") - - # init engine - self.token_processor = TritonTokenProcessor(self.cfg, self) - self.engine = engine.Engine(self.cfg, self.token_processor) - model_server_logger.info("Creat engine...") - self.engine.start() - model_server_logger.info("Create engine success") - - self._initialize_push_mode() - model_server_logger.info("Init triton server success") - - - def execute(self, requests): - """ - Triton service main function, - handling requests received by the Triton framework - """ - if len(requests) != 1: - raise pb_utils.TritonModelException( - "Only support batch=1, but now it's {}.".format(len(requests))) - request = requests[0] - current_response_sender = request.get_response_sender() - request_tensor = pb_utils.get_input_tensor_by_name(request, "IN") - tasks = json.loads(request_tensor.as_numpy()[0]) - - model_server_logger.info(f"receive task: {tasks}") - self._process_task_push_mode(tasks, current_response_sender) - self._update_metrics() - - def finalize(self): - """ - Triton service exit function - """ - model_server_logger.info("Triton service will be terminated...") - wait_time = 300 - while not self.engine.all_tasks_finished(): - if wait_time <= 0: - model_server_logger.warning(f"Ignore the unfinished tasks, force to stop.") - break - model_server_logger.info(f"There's unfinished tasks, wait {wait_time}...") - wait_time -= 5 - time.sleep(5) - model_server_logger.info("Terminate the engine now.") - self.enable_insert_task_push_mode = False - time.sleep(1) - del self.engine - if hasattr(self, "http_process"): - self.http_process.kill() - model_server_logger.info("Triton service is terminated!") - - def _initialize_push_mode(self): - from server.data.processor import DataProcessor - self.data_processor = DataProcessor() - model_server_logger.info("create data processor success") - - if self.cfg.push_mode_http_port < 0: - model_server_logger.info("HTTP server for push mode is disabled.") - else: - model_server_logger.info("launch http server...") - - current_dir_path = os.path.split(os.path.abspath(__file__))[0] - http_py_file = "app.py" - http_py_path = os.path.join(current_dir_path, "http_server", http_py_file) - http_cmd = f"python3 {http_py_path} --port={self.cfg.push_mode_http_port} " \ - f"--workers={self.cfg.push_mode_http_workers} >log/launch_http.log 2>&1" - - model_server_logger.info(f"Launch HTTP server for push mode, command:{http_cmd}") - self.http_process = subprocess.Popen(http_cmd, shell=True, preexec_fn=os.setsid) - time.sleep(3) - exit_code = self.http_process.poll() - if exit_code is None: - http_url = f"http://127.0.0.1:{self.cfg.push_mode_http_port}/v1/chat/completions" - model_server_logger.info(f"Launch HTTP server for push mode success, http_url:{http_url}") - else: - error_msg = "\n Launch HTTP service for push mode failed in 3 seconds. " \ - "Please check log/launch_http.log file \n" - model_server_logger.error(error_msg) - model_server_logger.info("init push server success") - - self.response_sender = dict() - self.task_info = dict() - self.cached_task_deque = deque() - self.enable_insert_task_push_mode = True - self.insert_task_to_engine_thread = threading.Thread( - target=self._insert_task_push_mode, args=()) - self.insert_task_to_engine_thread.daemon = True - self.insert_task_to_engine_thread.start() - - def _process_task_push_mode(self, tasks, current_response_sender): - """ - check request and insert into cached_task_deque - - Args: - tasks (list): list of request - current_response_sender: response sender for current request - """ - try: - tik = time.time() - req_id = tasks[0]["req_id"] - cached_task_num = len(self.cached_task_deque) - if cached_task_num >= self.cfg.max_cached_task_num: - error_msg = f"cached task num ({cached_task_num}) exceeds " \ - f"the limit ({self.cfg.max_cached_task_num})" - _send_error(error_msg, current_response_sender, req_id=req_id) - return - - if not tasks or len(tasks) != 1 or not tasks[0]: - error_msg = f"request data should not be empty and query " \ - f"num {len(tasks)} should be 1" - _send_error(error_msg, current_response_sender, req_id=req_id) - return - - task = tasks[0] - task["preprocess_start_time"] = datetime.now() - - error_msg = check_basic_params(task) - if error_msg != []: - _send_error(error_msg, current_response_sender, req_id=req_id) - return - - task_id = task["req_id"] - with self.thread_lock: - if task_id in self.response_sender: - error_msg = f"The req_id {task_id} already exists in the current batch, " \ - f"the current request will be ignored." - _send_error(error_msg, current_response_sender, req_id=req_id) - return - - task = add_default_params(task) - - if int(task.get("enable_text_truncate", 1)): - real_seq_len = self.cfg.max_seq_len - task.get("max_dec_len", 800) - task = self.data_processor.process_request(task, max_seq_len=real_seq_len) - else: - task = self.data_processor.process_request(task) - - input_ids_len = len(task["input_ids"]) - if "max_dec_len" not in task: - task["max_dec_len"] = min(self.cfg.max_seq_len - input_ids_len, self.cfg.dec_len_limit) - min_dec_len = task["min_dec_len"] - if input_ids_len + min_dec_len >= self.cfg.max_seq_len: - error_msg = f"Input text is too long, input_ids_len ({input_ids_len}) " \ - f"+ min_dec_len ({min_dec_len}) >= max_seq_len " - _send_error(error_msg, current_response_sender, req_id=req_id) - return - - if input_ids_len > self.cfg.seq_len_limit: - error_msg = f"Length of input token({input_ids_len}) exceeds the limit MAX_SEQ_LEN({self.cfg.seq_len_limit})." - _send_error(error_msg, current_response_sender, req_id=req_id) - return - if task["max_dec_len"] > self.cfg.dec_len_limit: - error_msg = f"The parameter max_dec_len({task['max_dec_len']}) exceeds the limit MAX_DEC_LEN({self.cfg.dec_len_limit})." - _send_error(error_msg, current_response_sender, req_id=req_id) - return - - required_block_num = self.engine.resource_manager.get_required_block_number(input_ids_len) - if required_block_num > self.engine.resource_manager.total_block_number(): - error_msg = f"The input task required resources is exceed the limit, task={task}." - _send_error(error_msg, current_response_sender, req_id=req_id) - return - - with self.thread_lock: - self.response_sender[task_id] = current_response_sender - self.task_info[task_id] = {"prompt_tokens": input_ids_len} - - task["preprocess_end_time"] = datetime.now() - self.cached_task_deque.appendleft(task) - tok = time.time() - model_server_logger.info(f"cache task with req_id ({task_id}), " - f"cost time: {tok-tik}s, cached_task_num: {len(self.cached_task_deque)}.") - model_server_logger.debug(f"cache task: {task}") - except Exception as e: - error_msg = "Unexcepted promblem happend while insert new task to server task queue: {}, {}".format( - e, str(traceback.format_exc())) - _send_error(error_msg, current_response_sender) - - def _insert_task_push_mode(self): - """ - Insert task to engine thread, monitor cached_task_deque. - if the engine has resource, insert task to engine - """ - try: - while self.enable_insert_task_push_mode: - if not hasattr(self, "engine") or self.engine is None: - time.sleep(0.1) - continue - if self.engine.available_batch() == 0: - time.sleep(0.001) - continue - if len(self.cached_task_deque) == 0: - time.sleep(0.001) - continue - if not self.engine.is_queue_empty(): - time.sleep(0.001) - continue - - i_bs = 0 - for _ in range(self.cfg.max_prefill_batch): - if len(self.cached_task_deque) == 0: - break - if self.engine.available_batch() == 0: - break - while i_bs < self.cfg.max_batch_size: - if self.engine.task_is_finished(i_bs): - break - i_bs += 1 - if i_bs >= self.cfg.max_batch_size: - break - input_token_num = len(self.cached_task_deque[-1]["input_ids"]) - if not self.engine.is_resource_sufficient(input_token_num): - break - task = self.cached_task_deque.pop() - try: - self.engine.insert_tasks([task]) - except Exception as e: - err_msg = "Error happend while insert task to engine: {}, {}.".format( - e, str(traceback.format_exc())) - with self.thread_lock: - _send_result({"error_msg": err_msg}, - self.response_sender[task["req_id"]], 1) - del self.response_sender[task["req_id"]] - model_server_logger.info("finish insert_task_push_mode thread") - except Exception as e: - model_server_logger.error("insert_task_push_mode thread exit " - f"unexpectedly, {e}. {str(traceback.format_exc())}") - - def _update_metrics(self): - """ - update metrics - """ - block_num = self.engine.available_block_num() - batch_size = self.engine.available_batch() - self.metrics["block_num"].set(block_num) - self.metrics["max_batch_size"].set(self.cfg.max_batch_size) - self.metrics["batch_size"].set(self.cfg.max_batch_size - batch_size) - self.metrics["max_block_num"].set(self.cfg.max_block_num) - self.metrics["available_resource"].set(block_num * 1.0 / - self.cfg.max_block_num) - - def _get_current_server_info(self): - """ - get server info - """ - available_batch_size = min(self.cfg.max_prefill_batch, - self.engine.available_batch()) - available_block_num = self.engine.available_block_num() - server_info = { - "block_size": int(self.cfg.block_size), - "block_num": int(available_block_num), - "dec_token_num": int(self.cfg.dec_token_num), - "available_resource": - 1.0 * available_block_num / self.cfg.max_block_num, - "max_batch_size": int(available_batch_size), - } - return server_info - - -def _send_result(result_dict, sender, end_flag=0): - """ - Send inference result - - Args: - result_dict (dict): result of inference - sender (grpc.aio.ServerReaderWriter): gRPC ServerReaderWriter object. - end_flag (int, optional): flag of end. Defaults to 0. - """ - response = None - if result_dict: - result_dict = json.dumps(result_dict) - end_output = pb_utils.Tensor("OUT", - np.array([result_dict], dtype=np.object_)) - response = pb_utils.InferenceResponse(output_tensors=[end_output]) - if response is None and end_flag == 0: - return - sender.send(response, flags=end_flag) - -def _send_error(error_msg, sender, error_code=200, req_id=None): - """ - Send error inference result - - Args: - error_msg (str): error message - sender (grpc.aio.ServerReaderWriter): gRPC ServerReaderWriter object. - error_code (int, optional): error code. Defaults to 200. - req_id (str, optional): request id. Defaults to None - """ - if not isinstance(error_msg, str): - error_msg = str(error_msg) - error_info = {"req_id": req_id, "error_msg": error_msg, "error_code": error_code, "version": "4.6", "timestamp": time.time()} - error_logger.info(f"{error_info}") - model_server_logger.error(error_msg) - _send_result(error_info, sender, 1) - - -TritonPythonModel = TritonServer diff --git a/services/llm/server/server/triton_server_helper.py b/services/llm/server/server/triton_server_helper.py deleted file mode 100644 index b299cd4204f..00000000000 --- a/services/llm/server/server/triton_server_helper.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import queue -import socket -import subprocess -import time -from collections import defaultdict -from multiprocessing import shared_memory - -import numpy as np -import uvicorn -from fastapi import FastAPI, HTTPException, Request -from fastapi.responses import JSONResponse, Response -from server.engine.config import Config -from server.utils import get_logger - -app = FastAPI() -env_config = Config() -logger = get_logger("health_checker", "health_checker.log") - - -@app.get("/v2/health/ready") -def check_health(): - """ - health check interface - """ - status, error_info = check() - if status is True: - logger.info("check_health: OK") - return Response() - else: - logger.info("check_health: Bad") - return JSONResponse( - status_code=500, - content=error_info) - - -@app.get("/v2/health/live") -def check_live(): - """ - health check interface - """ - status, error_info = check() - if status is True: - logger.info("check_health: OK") - return Response() - else: - logger.info("check_health: Bad") - return JSONResponse( - status_code=500, - content=error_info) - - -def check_infer_engine_process(): - """ - check if infer process is alive - - return: - status: bool, True if process is alive else False - """ - mp_num = int(env_config.mp_num) - for i in range(mp_num): - try: - infer_live_flag_shm = shared_memory.SharedMemory(name=env_config.get_unique_name("shm_flag_infer_{}_live".format(i))) - except Exception as e: - return False - return True - - -def check(): - """ - State detection interface for inference services - - return: - status: bool, True if process is alive else False - """ - error_info = {} - grpc_port = os.getenv("GRPC_PORT") - - # 1. check server is ready - if grpc_port is not None: - sock = socket.socket() - try: - sock.connect(('localhost', int(grpc_port))) - except Exception: - error_info["error_code"] = 1 - error_info["error_msg"] = "server is not ready" - logger.info("server is not ready") - return False, error_info - finally: - sock.close() - - # 2.check engine is ready - is_engine_live = check_infer_engine_process() - if is_engine_live is False: - error_info["error_code"] = 2 - error_info["error_msg"] = "infer engine is down" - logger.info("infer engine is down") - return False, error_info - - engine_ready_checker = np.ndarray(engine_ready_check_flag.shape, dtype=engine_ready_check_flag.dtype, - buffer=shm_engine_ready_check_flag.buf) - if engine_ready_checker[0] == 0: - error_info["error_code"] = 2 - error_info["error_msg"] = "infer engine is down" - logger.info("infer engine is down") - return False, error_info - - # check engine is hang - engine_hang_checker = np.ndarray(engine_healthy_recorded_time.shape, dtype=engine_healthy_recorded_time.dtype, - buffer=shm_engine_healthy_recorded_time.buf) - elapsed_time = time.time() - engine_hang_checker[0] - logger.info("engine_checker elapsed time: {}".format(elapsed_time)) - if (engine_hang_checker[0]) and (elapsed_time > time_interval_threashold): - error_info["error_code"] = 3 - error_info["error_msg"] = "infer engine hangs" - logger.info("infer engine hangs") - return False, error_info - - return True, error_info - - -def start_health_checker(http_port): - import sys - sys.stdout = open("log/health_http.log", 'w') - sys.stderr = sys.stdout - uvicorn.run(app=app, host='0.0.0.0', port=http_port, workers=1, log_level="info") - - -# if infer engine not update for more than 10 seconds，consider it as hang or dead -time_interval_threashold = env_config.check_health_interval -engine_healthy_recorded_time = np.zeros([1], dtype=float) - -shm_engine_healthy_recorded_time = shared_memory.SharedMemory( - create=True, - size=engine_healthy_recorded_time.nbytes, - name=env_config.get_unique_name("engine_healthy_recorded_time")) - -engine_ready_check_flag = np.zeros([1], dtype=np.int32) -shm_engine_ready_check_flag = shared_memory.SharedMemory( - create=True, - size=engine_ready_check_flag.nbytes, - name=env_config.get_unique_name("engine_ready_check_flag")) diff --git a/services/llm/server/server/utils.py b/services/llm/server/server/utils.py deleted file mode 100644 index bb80f6b0a47..00000000000 --- a/services/llm/server/server/utils.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import codecs -import logging -import os -import pickle -import re -import subprocess -import time -from datetime import datetime -from enum import Enum -from logging.handlers import BaseRotatingHandler -from pathlib import Path - - -class DailyRotatingFileHandler(BaseRotatingHandler): - """ - like `logging.TimedRotatingFileHandler`, but this class support multi-process - """ - - def __init__( - self, - filename, - backupCount=0, - encoding="utf-8", - delay=False, - utc=False, - **kwargs - ): - self.backup_count = backupCount - self.utc = utc - self.suffix = "%Y-%m-%d" - self.base_log_path = Path(filename) - self.base_filename = self.base_log_path.name - self.current_filename = self._compute_fn() - self.current_log_path = self.base_log_path.with_name(self.current_filename) - BaseRotatingHandler.__init__(self, filename, "a", encoding, delay) - - def shouldRollover(self, record): - """ - check scroll through the log - """ - if self.current_filename != self._compute_fn(): - return True - return False - - def doRollover(self): - """ - scroll log - """ - if self.stream: - self.stream.close() - self.stream = None - - self.current_filename = self._compute_fn() - self.current_log_path = self.base_log_path.with_name(self.current_filename) - - if not self.delay: - self.stream = self._open() - - self.delete_expired_files() - - def _compute_fn(self): - """ - Calculate the log file name corresponding current time - """ - return self.base_filename + "." + time.strftime(self.suffix, time.localtime()) - - def _open(self): - """ - open new log file - """ - if self.encoding is None: - stream = open(str(self.current_log_path), self.mode) - else: - stream = codecs.open(str(self.current_log_path), self.mode, self.encoding) - - if self.base_log_path.exists(): - try: - if ( - not self.base_log_path.is_symlink() - or os.readlink(self.base_log_path) != self.current_filename - ): - os.remove(self.base_log_path) - except OSError: - pass - - try: - os.symlink(self.current_filename, str(self.base_log_path)) - except OSError: - pass - return stream - - def delete_expired_files(self): - """ - delete expired log files - """ - if self.backup_count <= 0: - return - - file_names = os.listdir(str(self.base_log_path.parent)) - result = [] - prefix = self.base_filename + "." - plen = len(prefix) - for file_name in file_names: - if file_name[:plen] == prefix: - suffix = file_name[plen:] - if re.match(r"^\d{4}-\d{2}-\d{2}(\.\w+)?$", suffix): - result.append(file_name) - if len(result) < self.backup_count: - result = [] - else: - result.sort() - result = result[: len(result) - self.backup_count] - - for file_name in result: - os.remove(str(self.base_log_path.with_name(file_name))) - - -def get_logger(name, file_name, without_formater=False): - """ - get logger - """ - log_dir = os.getenv("FD_LOG_DIR", default="log") - is_debug = int(os.getenv("FD_DEBUG", default=0)) - logger = logging.getLogger(name) - if is_debug: - logger.setLevel(level=logging.DEBUG) - else: - logger.setLevel(level=logging.INFO) - - LOG_FILE = "{0}/{1}".format(log_dir, file_name) - backup_count = int(os.getenv("FD_LOG_BACKUP_COUNT", 7)) - handler = DailyRotatingFileHandler(LOG_FILE, backupCount=backup_count) - - formatter = logging.Formatter( - "%(levelname)-8s %(asctime)s %(process)-5s %(filename)s[line:%(lineno)d] %(message)s" - ) - if not without_formater: - handler.setFormatter(formatter) - logger.addHandler(handler) - handler.propagate = False - return logger - - -def str_to_datetime(date_string): - """ - string to datetime class object - """ - if "." in date_string: - return datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S.%f") - else: - return datetime.strptime(date_string, "%Y-%m-%d %H:%M:%S") - - -def datetime_diff(datetime_start, datetime_end): - """ - Calculate the difference between two dates and times(s) - - Args: - datetime_start (Union[str, datetime.datetime]): start time - datetime_end (Union[str, datetime.datetime]): end time - - Returns: - float: date time difference(s) - """ - if isinstance(datetime_start, str): - datetime_start = str_to_datetime(datetime_start) - if isinstance(datetime_end, str): - datetime_end = str_to_datetime(datetime_end) - if datetime_end > datetime_start: - cost = datetime_end - datetime_start - else: - cost = datetime_start - datetime_end - return cost.total_seconds() - - -model_server_logger = get_logger("model_server", "infer_server.log") -http_server_logger = get_logger("http_server", "http_server.log") -data_processor_logger = get_logger("data_processor", "data_processor.log") -monitor_logger = get_logger("monitor_logger", "monitor_logger.log", True) -error_logger = get_logger("error_logger", "error_logger.log", True) From 75d5e81b7cb6676fd2a3aa01344ee21fea979992 Mon Sep 17 00:00:00 2001 From: SherkeyXD <57581480+SherkeyXD@users.noreply.github.com> Date: Thu, 10 Sep 2026 18:00:57 +0800 Subject: [PATCH 4/5] chore: clean up dead cmake files, fix c_api path, update ci comments and build script --- .github/workflows/build.yml | 4 +- bindings/c_api/CMakeLists.txt | 8 +- cmake/config_cpack.cmake | 38 ------- cmake/onnxruntime.cmake | 129 ----------------------- fastdeploy/runtime/backends/ort/option.h | 7 ++ scripts/build.py | 10 +- 6 files changed, 19 insertions(+), 177 deletions(-) delete mode 100644 cmake/config_cpack.cmake delete mode 100644 cmake/onnxruntime.cmake diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f5164e31436..e1375a7ac99 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ concurrency: jobs: # ========================================================================= - # 1. macOS ARM64 (Apple Silicon: CoreML & CPU) + # 1. macOS ARM64 (Apple Silicon: CPU) # ========================================================================= build-macos-arm64: name: macOS (arm64) @@ -48,7 +48,7 @@ jobs: path: fastdeploy_ppocr-linux-x64.tar.gz # ========================================================================= - # 3. Windows x64 (DirectML & CPU) + # 3. Windows x64 (CPU) # ========================================================================= build-windows-x64: name: Windows (x64) diff --git a/bindings/c_api/CMakeLists.txt b/bindings/c_api/CMakeLists.txt index 2f4d56497d7..12f6ae45f89 100644 --- a/bindings/c_api/CMakeLists.txt +++ b/bindings/c_api/CMakeLists.txt @@ -19,11 +19,11 @@ if(NOT WITH_CAPI) return() endif() -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fastdeploy_capi/core/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/fastdeploy_capi/core/config.h) -file(GLOB_RECURSE DEPLOY_CAPI_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/fastdeploy_capi/*.cc) +configure_file(${CMAKE_CURRENT_LIST_DIR}/fastdeploy_capi/core/config.h.in ${CMAKE_CURRENT_LIST_DIR}/fastdeploy_capi/core/config.h) +file(GLOB_RECURSE DEPLOY_CAPI_SRCS ${CMAKE_CURRENT_LIST_DIR}/fastdeploy_capi/*.cc) if(NOT ENABLE_VISION) - file(GLOB_RECURSE DEPLOY_VISION_CAPI_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/fastdeploy_capi/vision/*.cc) + file(GLOB_RECURSE DEPLOY_VISION_CAPI_SRCS ${CMAKE_CURRENT_LIST_DIR}/fastdeploy_capi/vision/*.cc) list(REMOVE_ITEM DEPLOY_CAPI_SRCS ${DEPLOY_VISION_CAPI_SRCS}) endif() list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_CAPI_SRCS}) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_LIST_DIR}) diff --git a/cmake/config_cpack.cmake b/cmake/config_cpack.cmake deleted file mode 100644 index 284356d9231..00000000000 --- a/cmake/config_cpack.cmake +++ /dev/null @@ -1,38 +0,0 @@ -if(NOT UNIX) - return() -endif() - -set(PACKAGE_SYS_VERSION "linux") -if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") - set(PACKAGE_SYS_VERSION "${PACKAGE_SYS_VERSION}-aarch64") -else() - set(PACKAGE_SYS_VERSION "${PACKAGE_SYS_VERSION}-x64") -endif() -if(WITH_CUDA) - set(PACKAGE_SYS_VERSION "${PACKAGE_SYS_VERSION}-gpu") -endif() - -# set(CPACK_ERROR_ON_ABSOLUTE_INSTALL_DESTINATION ON) -set(CPACK_VERBATIM_VARIABLES TRUE) -set(CPACK_GENERATOR DEB RPM) -set(CPACK_THREADS 0) -set(CPACK_PACKAGE_CONTACT "fastdeploy@baidu.com") -set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") -set(CPACK_PACKAGE_VERSION "${FASTDEPLOY_VERSION}") -set(CPACK_PACKAGE_FILE_NAME "${PROJECT_NAME}-${PACKAGE_SYS_VERSION}-${FASTDEPLOY_VERSION}") -set(CPACK_PACKAGE_NAME "${PROJECT_NAME}") - -set(CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION TRUE) -configure_file(${CMAKE_CURRENT_LIST_DIR}/cpack/debian_postinst.in ${CMAKE_CURRENT_BINARY_DIR}/cpack/postinst @ONLY) -configure_file(${CMAKE_CURRENT_LIST_DIR}/cpack/debian_prerm.in ${CMAKE_CURRENT_BINARY_DIR}/cpack/prerm @ONLY) -set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA - "${CMAKE_CURRENT_BINARY_DIR}/cpack/postinst" - "${CMAKE_CURRENT_BINARY_DIR}/cpack/prerm") - -set(CPACK_RPM_PACKAGE_AUTOREQ FALSE) -configure_file(${CMAKE_CURRENT_LIST_DIR}/cpack/rpm_postinst.in ${CMAKE_CURRENT_BINARY_DIR}/cpack/rpm_postinst @ONLY) -configure_file(${CMAKE_CURRENT_LIST_DIR}/cpack/rpm_postrm.in ${CMAKE_CURRENT_BINARY_DIR}/cpack/rpm_postrm @ONLY) -set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/cpack/rpm_postinst") -set(CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/cpack/rpm_postrm") - -include(CPack) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake deleted file mode 100644 index 2bc32bd7b54..00000000000 --- a/cmake/onnxruntime.cmake +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -include(ExternalProject) - -set(ONNXRUNTIME_PROJECT "extern_onnxruntime") -set(ONNXRUNTIME_PREFIX_DIR ${THIRD_PARTY_PATH}/onnxruntime) -set(ONNXRUNTIME_SOURCE_DIR - ${THIRD_PARTY_PATH}/onnxruntime/src/${ONNXRUNTIME_PROJECT}) -set(ONNXRUNTIME_INSTALL_DIR ${THIRD_PARTY_PATH}/install/onnxruntime) - -if (ORT_DIRECTORY) - message(STATUS "Use the onnxruntime lib specified by user. The ONNXRuntime path: ${ORT_DIRECTORY}") - STRING(REGEX REPLACE "\\\\" "/" ORT_DIRECTORY ${ORT_DIRECTORY}) - set(ONNXRUNTIME_INC_DIR - "${ORT_DIRECTORY}/include" - CACHE PATH "onnxruntime include directory." FORCE) - - set(ONNXRUNTIME_LIB_DIR - "${ORT_DIRECTORY}/lib" - CACHE PATH "onnxruntime lib directory." FORCE) -else() - message(STATUS "Use the default onnxruntime lib. The ONNXRuntime path: ${ONNXRUNTIME_INSTALL_DIR}") - set(ONNXRUNTIME_INC_DIR - "${ONNXRUNTIME_INSTALL_DIR}/include" - CACHE PATH "onnxruntime include directory." FORCE) - set(ONNXRUNTIME_LIB_DIR - "${ONNXRUNTIME_INSTALL_DIR}/lib" - CACHE PATH "onnxruntime lib directory." FORCE) -endif() -set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${ONNXRUNTIME_LIB_DIR}") - -set(ONNXRUNTIME_VERSION "1.29.0") -set(ONNXRUNTIME_URL_PREFIX "https://bj.bcebos.com/paddle2onnx/libs/") - -if(WIN32) - if(WITH_CUDA) - set(ONNXRUNTIME_FILENAME "onnxruntime-win-x64-gpu-${ONNXRUNTIME_VERSION}.zip") - elseif(WITH_DIRECTML) - set(ONNXRUNTIME_FILENAME "onnxruntime-directml-win-x64.zip") - else() - set(ONNXRUNTIME_FILENAME "onnxruntime-win-x64-${ONNXRUNTIME_VERSION}.zip") - endif() - if(NOT CMAKE_CL_64) - if(WITH_DIRECTML) - set(ONNXRUNTIME_FILENAME "onnxruntime-directml-win-x86.zip") - else() - set(ONNXRUNTIME_FILENAME "onnxruntime-win-x86-${ONNXRUNTIME_VERSION}.zip") - endif() - endif() -elseif(APPLE) - if(CURRENT_OSX_ARCH MATCHES "arm64") - set(ONNXRUNTIME_FILENAME "onnxruntime-osx-arm64-${ONNXRUNTIME_VERSION}.tgz") - else() - set(ONNXRUNTIME_FILENAME "onnxruntime-osx-x86_64-${ONNXRUNTIME_VERSION}.tgz") - endif() -else() - if(WITH_CUDA) - if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") - message("Cannot compile with onnxruntime-gpu while in linux-aarch64 platform, fallback to onnxruntime-cpu") - set(ONNXRUNTIME_FILENAME "onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}.tgz") - else() - set(ONNXRUNTIME_FILENAME "onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}.tgz") - endif() - else() - if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") - set(ONNXRUNTIME_FILENAME "onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}.tgz") - else() - # cross-compling while the host is x64 but the target is aarch64. - if ((CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") OR (CMAKE_SYSTEM_PROCESSOR MATCHES "arm")) - set(ONNXRUNTIME_FILENAME "onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}.tgz") - else() - set(ONNXRUNTIME_FILENAME "onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz") - endif() - endif() - endif() -endif() -set(ONNXRUNTIME_URL "${ONNXRUNTIME_URL_PREFIX}${ONNXRUNTIME_FILENAME}") - -include_directories(${ONNXRUNTIME_INC_DIR} -)# For ONNXRUNTIME code to include internal headers. - -if(WIN32) - set(ONNXRUNTIME_LIB - "${ONNXRUNTIME_LIB_DIR}/onnxruntime.lib" - CACHE FILEPATH "ONNXRUNTIME shared library." FORCE) -elseif(APPLE) - set(ONNXRUNTIME_LIB - "${ONNXRUNTIME_LIB_DIR}/libonnxruntime.dylib" - CACHE FILEPATH "ONNXRUNTIME shared library." FORCE) -else() - set(ONNXRUNTIME_LIB - "${ONNXRUNTIME_LIB_DIR}/libonnxruntime.so" - CACHE FILEPATH "ONNXRUNTIME shared library." FORCE) -endif() - -if (NOT ORT_DIRECTORY) - ExternalProject_Add( - ${ONNXRUNTIME_PROJECT} - ${EXTERNAL_PROJECT_LOG_ARGS} - URL ${ONNXRUNTIME_URL} - PREFIX ${ONNXRUNTIME_PREFIX_DIR} - DOWNLOAD_NO_PROGRESS 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - UPDATE_COMMAND "" - INSTALL_COMMAND - ${CMAKE_COMMAND} -E remove_directory ${ONNXRUNTIME_INSTALL_DIR} && - ${CMAKE_COMMAND} -E make_directory ${ONNXRUNTIME_INSTALL_DIR} && - ${CMAKE_COMMAND} -E rename ${ONNXRUNTIME_SOURCE_DIR}/lib/ ${ONNXRUNTIME_INSTALL_DIR}/lib && - ${CMAKE_COMMAND} -E copy_directory ${ONNXRUNTIME_SOURCE_DIR}/include - ${ONNXRUNTIME_INC_DIR} - BUILD_BYPRODUCTS ${ONNXRUNTIME_LIB}) -endif() - -add_library(external_onnxruntime STATIC IMPORTED GLOBAL) -set_property(TARGET external_onnxruntime PROPERTY IMPORTED_LOCATION ${ONNXRUNTIME_LIB}) -add_dependencies(external_onnxruntime ${ONNXRUNTIME_PROJECT}) \ No newline at end of file diff --git a/fastdeploy/runtime/backends/ort/option.h b/fastdeploy/runtime/backends/ort/option.h index bb10608acaa..865a1e5fff5 100755 --- a/fastdeploy/runtime/backends/ort/option.h +++ b/fastdeploy/runtime/backends/ort/option.h @@ -59,6 +59,13 @@ struct OrtBackendOption { ort_disabled_ops_.insert(ort_disabled_ops_.end(), ops.begin(), ops.end()); } + /** \brief Callback to configure raw OrtSessionOptions directly. + * \note Complete Takeover Semantics: If this callback is provided, OrtBackend::BuildOption + * invokes it and returns immediately. All standard configurations in OrtBackendOption + * (e.g., intra/inter op threads, graph optimization level, execution providers like + * DirectML/CoreML/CUDA) will be completely bypassed. The caller is responsible for + * fully configuring the session_options. + */ bool (*configure_session_callback)(OrtSessionOptions* session_options, void* user_data) = nullptr; void* configure_session_callback_data = nullptr; }; diff --git a/scripts/build.py b/scripts/build.py index c91b6f48b9d..e2d8027d914 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -3,7 +3,7 @@ """ Cross-platform build and packaging script for FastDeploy (fastdeploy_ppocr). Usage: - python scripts/build.py --target [macos-arm64 | linux-x64 | windows-x64 | windows-arm64 | android-arm64 | python-wheel] + python scripts/build.py --target [macos-arm64 | linux-x64 | windows-x64 | android-arm64] """ import os @@ -29,7 +29,7 @@ def run_cmd(cmd, cwd=PROJECT_ROOT, env=None): cmd_env = os.environ.copy() if env: cmd_env.update(env) - ret = subprocess.run(cmd, cwd=cwd, env=cmd_env, shell=isinstance(cmd, str)) + ret = subprocess.run(cmd, cwd=cwd, env=cmd_env, shell=False) if ret.returncode != 0: print(f"\n[build.py] Error: command failed with return code {ret.returncode}", file=sys.stderr) sys.exit(ret.returncode) @@ -53,7 +53,10 @@ def extract_archive(archive_path, extract_to): zip_ref.extractall(extract_to) elif archive_path.endswith(".tar.gz") or archive_path.endswith(".tgz"): with tarfile.open(archive_path, 'r:gz') as tar_ref: - tar_ref.extractall(extract_to) + try: + tar_ref.extractall(extract_to, filter='data') + except TypeError: + tar_ref.extractall(extract_to) else: # Fallback for 7z / exe on Windows if shutil.which("7z"): @@ -71,7 +74,6 @@ def package_dist(target_name, fmt="tar.gz"): item_path = os.path.join(DIST_DIR, item) tar.add(item_path, arcname=item) elif fmt == "zip": - archive_file = os.path.join(PROJECT_ROOT, f"{pkg_name}.zip") shutil.make_archive(os.path.join(PROJECT_ROOT, pkg_name), 'zip', DIST_DIR) log(f"Artifact created: {pkg_name}.{fmt}") From 094dcf7866f43ce2b041355b0358f40dc84e2f4c Mon Sep 17 00:00:00 2001 From: Aliothmoon Date: Fri, 11 Sep 2026 01:50:08 +0800 Subject: [PATCH 5/5] fix: fail instead of falling back to unfiltered tar extraction --- scripts/build.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/build.py b/scripts/build.py index e2d8027d914..a2b54a886fa 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -52,11 +52,14 @@ def extract_archive(archive_path, extract_to): with zipfile.ZipFile(archive_path, 'r') as zip_ref: zip_ref.extractall(extract_to) elif archive_path.endswith(".tar.gz") or archive_path.endswith(".tgz"): + # Never fall back to unfiltered extraction: it allows path traversal (e.g. "../file") + if not hasattr(tarfile, "data_filter"): + raise RuntimeError( + "Safe tar extraction requires Python 3.12+ " + "(or 3.8.17+ / 3.9.17+ / 3.10.12+ / 3.11.4+ with the tarfile filter backport), " + f"current: {sys.version.split()[0]}") with tarfile.open(archive_path, 'r:gz') as tar_ref: - try: - tar_ref.extractall(extract_to, filter='data') - except TypeError: - tar_ref.extractall(extract_to) + tar_ref.extractall(extract_to, filter='data') else: # Fallback for 7z / exe on Windows if shutil.which("7z"):