Skip to content

Commit 0172f18

Browse files
xmfcxxmfcx
committed
refactor(ansible): drive the model downloads with one loop task
All 20 hf download tasks in the artifacts role were identical except for the repository, the tag and the local directory. One task now loops over a table of {repo, revision, dest}, with dest defaulting to the repository name. The load-bearing comments stay on the table rows. The 20 rendered commands, the environment and the destinations are unchanged. One behavior changes: a failed download no longer stops the bundles after it. Ansible runs every loop item, then reports the task as failed, so the play still stops at the task. Signed-off-by: Mete Fatih Cırıt <mfc@autoware.org>
1 parent f89cb5d commit 0172f18

2 files changed

Lines changed: 42 additions & 283 deletions

File tree

‎ansible/roles/artifacts/meta/main.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
dependencies:
2-
# Runs first: the download tasks below write into data_dir as the user, not as root.
2+
# Runs first: the download task below writes into data_dir as the user, not as root.
33
- role: autoware.dev_env.autoware_data_ownership
44
- role: autoware.dev_env.huggingface_cli

‎ansible/roles/artifacts/tasks/main.yaml‎

Lines changed: 41 additions & 282 deletions
Original file line numberDiff line numberDiff line change
@@ -1,282 +1,41 @@
1-
# yabloc_pose_initializer
2-
# v0.1 is the complete upstream PINTO export, the same file set the old archive delivered. Pruning it to the
3-
# files the node can actually use (tag v1.0) is a separate follow-up.
4-
- name: Download yabloc_pose_initializer model bundle ({{ revision }})
5-
vars:
6-
revision: v0.1
7-
ansible.builtin.command:
8-
cmd: >-
9-
hf download AutowareFoundation/yabloc_pose_initializer
10-
--revision {{ revision }}
11-
--local-dir "{{ data_dir }}/yabloc_pose_initializer"
12-
environment:
13-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
14-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
15-
changed_when: false
16-
17-
# bevfusion
18-
- name: Download bevfusion model bundle ({{ revision }})
19-
vars:
20-
revision: v2.0
21-
ansible.builtin.command:
22-
cmd: >-
23-
hf download AutowareFoundation/bevfusion
24-
--revision {{ revision }}
25-
--local-dir "{{ data_dir }}/bevfusion"
26-
environment:
27-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
28-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
29-
changed_when: false
30-
31-
# camera_streampetr
32-
- name: Download camera_streampetr model bundle ({{ revision }})
33-
vars:
34-
revision: v1.0
35-
ansible.builtin.command:
36-
cmd: >-
37-
hf download AutowareFoundation/camera_streampetr
38-
--revision {{ revision }}
39-
--local-dir "{{ data_dir }}/camera_streampetr"
40-
environment:
41-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
42-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
43-
changed_when: false
44-
45-
# bevdet
46-
- name: Download tensorrt_bevdet model bundle ({{ revision }})
47-
vars:
48-
revision: v1.0
49-
ansible.builtin.command:
50-
cmd: >-
51-
hf download AutowareFoundation/tensorrt_bevdet
52-
--revision {{ revision }}
53-
--local-dir "{{ data_dir }}/tensorrt_bevdet"
54-
environment:
55-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
56-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
57-
changed_when: false
58-
59-
# image_projection_based_fusion
60-
- name: Download image_projection_based_fusion model bundle ({{ revision }})
61-
vars:
62-
revision: v5.0
63-
ansible.builtin.command:
64-
cmd: >-
65-
hf download AutowareFoundation/image_projection_based_fusion
66-
--revision {{ revision }}
67-
--local-dir "{{ data_dir }}/image_projection_based_fusion"
68-
environment:
69-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
70-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
71-
changed_when: false
72-
73-
# lidar_apollo_instance_segmentation
74-
- name: Download lidar_apollo_instance_segmentation model bundle ({{ revision }})
75-
vars:
76-
revision: v1.0
77-
ansible.builtin.command:
78-
cmd: >-
79-
hf download AutowareFoundation/lidar_apollo_instance_segmentation
80-
--revision {{ revision }}
81-
--local-dir "{{ data_dir }}/lidar_apollo_instance_segmentation"
82-
environment:
83-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
84-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
85-
changed_when: false
86-
87-
# lidar_centerpoint
88-
- name: Download lidar_centerpoint model bundle ({{ revision }})
89-
vars:
90-
revision: v4.0
91-
ansible.builtin.command:
92-
cmd: >-
93-
hf download AutowareFoundation/lidar_centerpoint
94-
--revision {{ revision }}
95-
--local-dir "{{ data_dir }}/lidar_centerpoint"
96-
environment:
97-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
98-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
99-
changed_when: false
100-
101-
# lidar_transfusion
102-
- name: Download lidar_transfusion model bundle ({{ revision }})
103-
vars:
104-
revision: v2.1
105-
ansible.builtin.command:
106-
cmd: >-
107-
hf download AutowareFoundation/lidar_transfusion
108-
--revision {{ revision }}
109-
--local-dir "{{ data_dir }}/lidar_transfusion"
110-
environment:
111-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
112-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
113-
changed_when: false
114-
115-
# tensorrt_yolox
116-
# Includes the whole_image_traffic_light_detector models (source tl_detector_yolox_s/v1).
117-
- name: Download tensorrt_yolox model bundle ({{ revision }})
118-
vars:
119-
revision: v1.0
120-
ansible.builtin.command:
121-
cmd: >-
122-
hf download AutowareFoundation/tensorrt_yolox
123-
--revision {{ revision }}
124-
--local-dir "{{ data_dir }}/tensorrt_yolox"
125-
environment:
126-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
127-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
128-
changed_when: false
129-
130-
# traffic_light_classifier
131-
- name: Download traffic_light_classifier model bundle ({{ revision }})
132-
vars:
133-
revision: v4.0
134-
ansible.builtin.command:
135-
cmd: >-
136-
hf download AutowareFoundation/traffic_light_classifier
137-
--revision {{ revision }}
138-
--local-dir "{{ data_dir }}/traffic_light_classifier"
139-
environment:
140-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
141-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
142-
changed_when: false
143-
144-
# diffusion_planner
145-
# Each version is a tag on one repository, and every tag holds its files at the repository root,
146-
# so the versions are downloaded separately into the per-version directories the node expects.
147-
- name: Download diffusion_planner model bundle ({{ revision }})
148-
vars:
149-
revision: v3.0
150-
ansible.builtin.command:
151-
cmd: >-
152-
hf download AutowareFoundation/diffusion_planner
153-
--revision {{ revision }}
154-
--local-dir "{{ data_dir }}/diffusion_planner/{{ revision }}"
155-
environment:
156-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
157-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
158-
changed_when: false
159-
160-
- name: Download diffusion_planner model bundle ({{ revision }})
161-
vars:
162-
revision: v3.1
163-
ansible.builtin.command:
164-
cmd: >-
165-
hf download AutowareFoundation/diffusion_planner
166-
--revision {{ revision }}
167-
--local-dir "{{ data_dir }}/diffusion_planner/{{ revision }}"
168-
environment:
169-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
170-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
171-
changed_when: false
172-
173-
- name: Download diffusion_planner model bundle ({{ revision }})
174-
vars:
175-
revision: v4.0
176-
ansible.builtin.command:
177-
cmd: >-
178-
hf download AutowareFoundation/diffusion_planner
179-
--revision {{ revision }}
180-
--local-dir "{{ data_dir }}/diffusion_planner/{{ revision }}"
181-
environment:
182-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
183-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
184-
changed_when: false
185-
186-
- name: Download diffusion_planner model bundle ({{ revision }})
187-
vars:
188-
revision: v5.0
189-
ansible.builtin.command:
190-
cmd: >-
191-
hf download AutowareFoundation/diffusion_planner
192-
--revision {{ revision }}
193-
--local-dir "{{ data_dir }}/diffusion_planner/{{ revision }}"
194-
environment:
195-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
196-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
197-
changed_when: false
198-
199-
# tensorrt_vad
200-
# The local directory is vad, not tensorrt_vad: the node reads $(var data_path)/vad/v0.1.
201-
- name: Download tensorrt_vad model bundle ({{ revision }})
202-
vars:
203-
revision: v0.1
204-
ansible.builtin.command:
205-
cmd: >-
206-
hf download AutowareFoundation/tensorrt_vad
207-
--revision {{ revision }}
208-
--local-dir "{{ data_dir }}/vad/{{ revision }}"
209-
environment:
210-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
211-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
212-
changed_when: false
213-
214-
# traffic_light_fine_detector
215-
- name: Download traffic_light_fine_detector model bundle ({{ revision }})
216-
vars:
217-
revision: v3.0
218-
ansible.builtin.command:
219-
cmd: >-
220-
hf download AutowareFoundation/traffic_light_fine_detector
221-
--revision {{ revision }}
222-
--local-dir "{{ data_dir }}/traffic_light_fine_detector"
223-
environment:
224-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
225-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
226-
changed_when: false
227-
228-
# simpl_prediction
229-
- name: Download simpl_prediction model bundle ({{ revision }})
230-
vars:
231-
revision: v0.1
232-
ansible.builtin.command:
233-
cmd: >-
234-
hf download AutowareFoundation/simpl_prediction
235-
--revision {{ revision }}
236-
--local-dir "{{ data_dir }}/simpl_prediction"
237-
environment:
238-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
239-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
240-
changed_when: false
241-
242-
# ptv3
243-
- name: Download ptv3 model bundle ({{ revision }})
244-
vars:
245-
revision: v4.0
246-
ansible.builtin.command:
247-
cmd: >-
248-
hf download AutowareFoundation/ptv3
249-
--revision {{ revision }}
250-
--local-dir "{{ data_dir }}/ptv3"
251-
environment:
252-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
253-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
254-
changed_when: false
255-
256-
# lidar_frnet
257-
- name: Download lidar_frnet model bundle ({{ revision }})
258-
vars:
259-
revision: v2.0
260-
ansible.builtin.command:
261-
cmd: >-
262-
hf download AutowareFoundation/lidar_frnet
263-
--revision {{ revision }}
264-
--local-dir "{{ data_dir }}/lidar_frnet"
265-
environment:
266-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
267-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
268-
changed_when: false
269-
270-
# calibration_status_classifier
271-
- name: Download calibration_status_classifier model bundle ({{ revision }})
272-
vars:
273-
revision: v2.0
274-
ansible.builtin.command:
275-
cmd: >-
276-
hf download AutowareFoundation/calibration_status_classifier
277-
--revision {{ revision }}
278-
--local-dir "{{ data_dir }}/calibration_status_classifier"
279-
environment:
280-
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
281-
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
282-
changed_when: false
1+
# dest is relative to data_dir and defaults to the repository name.
2+
- name: Download the model bundles from Hugging Face
3+
ansible.builtin.command:
4+
cmd: >-
5+
hf download AutowareFoundation/{{ item.repo }}
6+
--revision {{ item.revision }}
7+
--local-dir "{{ data_dir }}/{{ item.dest | default(item.repo) }}"
8+
environment:
9+
PATH: "{{ ansible_facts.env.HOME }}/.local/bin:{{ ansible_facts.env.PATH }}"
10+
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') }}"
11+
changed_when: false
12+
loop_control:
13+
label: "{{ item.repo }} {{ item.revision }}"
14+
loop:
15+
# v0.1 is the complete upstream PINTO export, the same file set the old archive delivered.
16+
# Pruning it to the files the node can actually use (tag v1.0) is a separate follow-up.
17+
- { repo: yabloc_pose_initializer, revision: v0.1 }
18+
- { repo: bevfusion, revision: v2.0 }
19+
- { repo: camera_streampetr, revision: v1.0 }
20+
- { repo: tensorrt_bevdet, revision: v1.0 }
21+
- { repo: image_projection_based_fusion, revision: v5.0 }
22+
- { repo: lidar_apollo_instance_segmentation, revision: v1.0 }
23+
- { repo: lidar_centerpoint, revision: v4.0 }
24+
- { repo: lidar_transfusion, revision: v2.1 }
25+
# Also holds the whole_image_traffic_light_detector models.
26+
- { repo: tensorrt_yolox, revision: v1.0 }
27+
- { repo: traffic_light_classifier, revision: v4.0 }
28+
# Each diffusion_planner version is a tag on one repository. Every tag holds the same
29+
# filenames at the repository root, so the tags need separate directories or they
30+
# overwrite each other.
31+
- { repo: diffusion_planner, revision: v3.0, dest: diffusion_planner/v3.0 }
32+
- { repo: diffusion_planner, revision: v3.1, dest: diffusion_planner/v3.1 }
33+
- { repo: diffusion_planner, revision: v4.0, dest: diffusion_planner/v4.0 }
34+
- { repo: diffusion_planner, revision: v5.0, dest: diffusion_planner/v5.0 }
35+
# The local directory is vad, not tensorrt_vad: the node reads $(var data_path)/vad/v0.1.
36+
- { repo: tensorrt_vad, revision: v0.1, dest: vad/v0.1 }
37+
- { repo: traffic_light_fine_detector, revision: v3.0 }
38+
- { repo: simpl_prediction, revision: v0.1 }
39+
- { repo: ptv3, revision: v4.0 }
40+
- { repo: lidar_frnet, revision: v2.0 }
41+
- { repo: calibration_status_classifier, revision: v2.0 }

0 commit comments

Comments
 (0)