Skip to content

Commit 4b1a924

Browse files
artem-astafevartem-astafev
committed
Added docker compose example for AMD ROCm deployment
Signed-off-by: artem-astafev <a.astafev@datamonsters.com>
1 parent cc108b5 commit 4b1a924

5 files changed

Lines changed: 340 additions & 0 deletions

File tree

‎AgentQnA/docker_compose/amd/gpu/rocm/compose.yaml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
services:
2+
agent-tgi-server:
3+
image: ${AGENTQNA_TGI_IMAGE}
4+
container_name: agent-tgi-server
5+
ports:
6+
- "${AGENTQNA_TGI_SERVICE_PORT-8085}:80"
7+
volumes:
8+
- /var/opea/agent-service/:/data
9+
environment:
10+
no_proxy: ${no_proxy}
11+
http_proxy: ${http_proxy}
12+
https_proxy: ${https_proxy}
13+
TGI_LLM_ENDPOINT: "http://${HOST_IP}:${AGENTQNA_TGI_SERVICE_PORT}"
14+
HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
15+
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
16+
shm_size: 1g
17+
devices:
18+
- /dev/kfd:/dev/kfd
19+
- /dev/dri/${AGENTQNA_CARD_ID}:/dev/dri/${AGENTQNA_CARD_ID}
20+
- /dev/dri/${AGENTQNA_RENDER_ID}:/dev/dri/${AGENTQNA_RENDER_ID}
21+
cap_add:
22+
- SYS_PTRACE
23+
group_add:
24+
- video
25+
security_opt:
26+
- seccomp:unconfined
27+
ipc: host
28+
command: --model-id ${LLM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192
29+
30+
worker-rag-agent:
31+
image: opea/agent-langchain:latest
32+
container_name: rag-agent-endpoint
33+
volumes:
34+
# - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/
35+
- ${TOOLSET_PATH}:/home/user/tools/
36+
ports:
37+
- "9095:9095"
38+
ipc: host
39+
environment:
40+
ip_address: ${ip_address}
41+
strategy: rag_agent_llama
42+
recursion_limit: ${recursion_limit_worker}
43+
llm_engine: tgi
44+
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
45+
llm_endpoint_url: ${LLM_ENDPOINT_URL}
46+
model: ${LLM_MODEL_ID}
47+
temperature: ${temperature}
48+
max_new_tokens: ${max_new_tokens}
49+
streaming: false
50+
tools: /home/user/tools/worker_agent_tools.yaml
51+
require_human_feedback: false
52+
RETRIEVAL_TOOL_URL: ${RETRIEVAL_TOOL_URL}
53+
no_proxy: ${no_proxy}
54+
http_proxy: ${http_proxy}
55+
https_proxy: ${https_proxy}
56+
LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
57+
LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
58+
LANGCHAIN_PROJECT: "opea-worker-agent-service"
59+
port: 9095
60+
61+
supervisor-react-agent:
62+
image: opea/agent-langchain:latest
63+
container_name: react-agent-endpoint
64+
depends_on:
65+
- agent-tgi-server
66+
- worker-rag-agent
67+
volumes:
68+
# - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/
69+
- ${TOOLSET_PATH}:/home/user/tools/
70+
ports:
71+
- "${AGENTQNA_FRONTEND_PORT}:9090"
72+
ipc: host
73+
environment:
74+
ip_address: ${ip_address}
75+
strategy: react_langgraph
76+
recursion_limit: ${recursion_limit_supervisor}
77+
llm_engine: tgi
78+
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
79+
llm_endpoint_url: ${LLM_ENDPOINT_URL}
80+
model: ${LLM_MODEL_ID}
81+
temperature: ${temperature}
82+
max_new_tokens: ${max_new_tokens}
83+
streaming: false
84+
tools: /home/user/tools/supervisor_agent_tools.yaml
85+
require_human_feedback: false
86+
no_proxy: ${no_proxy}
87+
http_proxy: ${http_proxy}
88+
https_proxy: ${https_proxy}
89+
LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
90+
LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
91+
LANGCHAIN_PROJECT: "opea-supervisor-agent-service"
92+
CRAG_SERVER: $CRAG_SERVER
93+
WORKER_AGENT_URL: $WORKER_AGENT_URL
94+
port: 9090

‎AgentQnA/docker_compose/amd/gpu/rocm/launch_agent_service_tgi_rocm.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Copyright (C) 2024 Advanced Micro Devices, Inc.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
WORKPATH=$(dirname "$PWD")/..
5+
export ip_address=${host_ip}
6+
export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
7+
export AGENTQNA_TGI_IMAGE=ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
8+
export AGENTQNA_TGI_SERVICE_PORT="8085"
9+
10+
# LLM related environment variables
11+
export AGENTQNA_CARD_ID="card1"
12+
export AGENTQNA_RENDER_ID="renderD136"
13+
export HF_CACHE_DIR=${HF_CACHE_DIR}
14+
ls $HF_CACHE_DIR
15+
export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
16+
#export NUM_SHARDS=4
17+
export LLM_ENDPOINT_URL="http://${ip_address}:${AGENTQNA_TGI_SERVICE_PORT}"
18+
export temperature=0.01
19+
export max_new_tokens=512
20+
21+
# agent related environment variables
22+
export AGENTQNA_WORKER_AGENT_SERVICE_PORT="9095"
23+
export TOOLSET_PATH=/home/huggingface/datamonsters/amd-opea/GenAIExamples/AgentQnA/tools/
24+
echo "TOOLSET_PATH=${TOOLSET_PATH}"
25+
export recursion_limit_worker=12
26+
export recursion_limit_supervisor=10
27+
export WORKER_AGENT_URL="http://${ip_address}:${AGENTQNA_WORKER_AGENT_SERVICE_PORT}/v1/chat/completions"
28+
export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
29+
export CRAG_SERVER=http://${ip_address}:18881
30+
31+
export AGENTQNA_FRONTEND_PORT="9090"
32+
33+
#retrieval_tool
34+
export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
35+
export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
36+
export REDIS_URL="redis://${host_ip}:26379"
37+
export INDEX_NAME="rag-redis"
38+
export MEGA_SERVICE_HOST_IP=${host_ip}
39+
export EMBEDDING_SERVICE_HOST_IP=${host_ip}
40+
export RETRIEVER_SERVICE_HOST_IP=${host_ip}
41+
export RERANK_SERVICE_HOST_IP=${host_ip}
42+
export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
43+
export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep"
44+
export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_file"
45+
export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/delete_file"
46+
47+
docker compose -f compose.yaml up -d
48+
49+

‎AgentQnA/docker_compose/amd/gpu/rocm/set_env.sh‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (C) 2024 Advanced Micro Devices, Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
6+
WORKPATH=$(dirname "$PWD")/..
7+
export ip_address=${host_ip}
8+
export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
9+
export AGENTQNA_TGI_IMAGE=ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
10+
export AGENTQNA_TGI_SERVICE_PORT="19001"
11+
12+
# LLM related environment variables
13+
export AGENTQNA_CARD_ID="card1"
14+
export AGENTQNA_RENDER_ID="renderD136"
15+
export HF_CACHE_DIR=${HF_CACHE_DIR}
16+
ls $HF_CACHE_DIR
17+
export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
18+
export NUM_SHARDS=4
19+
export LLM_ENDPOINT_URL="http://${ip_address}:${AGENTQNA_TGI_SERVICE_PORT}"
20+
export temperature=0.01
21+
export max_new_tokens=512
22+
23+
# agent related environment variables
24+
export AGENTQNA_WORKER_AGENT_SERVICE_PORT="9095"
25+
export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
26+
export recursion_limit_worker=12
27+
export recursion_limit_supervisor=10
28+
export WORKER_AGENT_URL="http://${ip_address}:${AGENTQNA_WORKER_AGENT_SERVICE_PORT}/v1/chat/completions"
29+
export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
30+
export CRAG_SERVER=http://${ip_address}:18881
31+
32+
export AGENTQNA_FRONTEND_PORT="15557"
33+
34+
#retrieval_tool
35+
export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
36+
export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
37+
export REDIS_URL="redis://${host_ip}:26379"
38+
export INDEX_NAME="rag-redis"
39+
export MEGA_SERVICE_HOST_IP=${host_ip}
40+
export EMBEDDING_SERVICE_HOST_IP=${host_ip}
41+
export RETRIEVER_SERVICE_HOST_IP=${host_ip}
42+
export RERANK_SERVICE_HOST_IP=${host_ip}
43+
export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
44+
export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep"
45+
export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_file"
46+
export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/delete_file"

‎AgentQnA/tests/step4a_launch_and_validate_agent_tgi_on_rocm.sh‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Intel Corporation
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -ex
6+
7+
WORKPATH=$(dirname "$PWD")
8+
export WORKDIR=$WORKPATH/../../
9+
echo "WORKDIR=${WORKDIR}"
10+
export ip_address=$(hostname -I | awk '{print $1}')
11+
export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
12+
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
13+
14+
export HF_CACHE_DIR=$WORKDIR/hf_cache
15+
if [ ! -d "$HF_CACHE_DIR" ]; then
16+
mkdir -p "$HF_CACHE_DIR"
17+
fi
18+
ls $HF_CACHE_DIR
19+
20+
21+
function start_agent_and_api_server() {
22+
echo "Starting CRAG server"
23+
docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
24+
25+
echo "Starting Agent services"
26+
cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/amd/gpu/rocm
27+
bash launch_agent_service_tgi_rocm.sh
28+
}
29+
30+
function validate() {
31+
local CONTENT="$1"
32+
local EXPECTED_RESULT="$2"
33+
local SERVICE_NAME="$3"
34+
35+
if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
36+
echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT"
37+
echo 0
38+
else
39+
echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
40+
echo 1
41+
fi
42+
}
43+
44+
function validate_agent_service() {
45+
echo "----------------Test agent ----------------"
46+
local CONTENT=$(http_proxy="" curl http://${ip_address}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
47+
"query": "Tell me about Michael Jackson song thriller"
48+
}')
49+
local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint")
50+
docker logs rag-agent-endpoint
51+
if [ "$EXIT_CODE" == "1" ]; then
52+
exit 1
53+
fi
54+
55+
local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
56+
"query": "Tell me about Michael Jackson song thriller"
57+
}')
58+
local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint")
59+
docker logs react-agent-endpoint
60+
if [ "$EXIT_CODE" == "1" ]; then
61+
exit 1
62+
fi
63+
64+
}
65+
66+
function main() {
67+
echo "==================== Start agent ===================="
68+
start_agent_and_api_server
69+
echo "==================== Agent started ===================="
70+
71+
echo "==================== Validate agent service ===================="
72+
validate_agent_service
73+
echo "==================== Agent service validated ===================="
74+
}
75+
76+
main

‎AgentQnA/tests/test_compose_on_rocm.sh‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Advanced Micro Devices, Inc.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -e
6+
7+
WORKPATH=$(dirname "$PWD")
8+
export WORKDIR=$WORKPATH/../../
9+
echo "WORKDIR=${WORKDIR}"
10+
export ip_address=$(hostname -I | awk '{print $1}')
11+
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
12+
export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
13+
14+
function stop_crag() {
15+
cid=$(docker ps -aq --filter "name=kdd-cup-24-crag-service")
16+
echo "Stopping container kdd-cup-24-crag-service with cid $cid"
17+
if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
18+
}
19+
20+
function stop_agent_docker() {
21+
cd $WORKPATH/docker_compose/amd/gpu/rocm
22+
# docker compose -f compose.yaml down
23+
container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2)
24+
for container_name in $container_list; do
25+
cid=$(docker ps -aq --filter "name=$container_name")
26+
echo "Stopping container $container_name"
27+
if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
28+
done
29+
}
30+
31+
function stop_retrieval_tool() {
32+
echo "Stopping Retrieval tool"
33+
local RETRIEVAL_TOOL_PATH=$WORKPATH/../DocIndexRetriever
34+
cd $RETRIEVAL_TOOL_PATH/docker_compose/intel/cpu/xeon/
35+
# docker compose -f compose.yaml down
36+
container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2)
37+
for container_name in $container_list; do
38+
cid=$(docker ps -aq --filter "name=$container_name")
39+
echo "Stopping container $container_name"
40+
if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
41+
done
42+
}
43+
echo "workpath: $WORKPATH"
44+
echo "=================== Stop containers ===================="
45+
stop_crag
46+
stop_agent_docker
47+
stop_retrieval_tool
48+
49+
cd $WORKPATH/tests
50+
51+
echo "=================== #1 Building docker images===================="
52+
bash step1_build_images.sh
53+
echo "=================== #1 Building docker images completed===================="
54+
55+
echo "=================== #2 Start retrieval tool===================="
56+
bash step2_start_retrieval_tool.sh
57+
echo "=================== #2 Retrieval tool started===================="
58+
59+
echo "=================== #3 Ingest data and validate retrieval===================="
60+
bash step3_ingest_data_and_validate_retrieval.sh
61+
echo "=================== #3 Data ingestion and validation completed===================="
62+
63+
echo "=================== #4 Start agent and API server===================="
64+
bash step4a_launch_and_validate_agent_tgi_on_rocm.sh
65+
echo "=================== #4 Agent test passed ===================="
66+
67+
echo "=================== #5 Stop agent and API server===================="
68+
stop_crag
69+
stop_agent_docker
70+
stop_retrieval_tool
71+
echo "=================== #5 Agent and API server stopped===================="
72+
73+
echo y | docker system prune
74+
75+
echo "ALL DONE!"

0 commit comments

Comments
 (0)