Skip to content

Commit fd0efa1

Browse files
authored
Merge pull request #405 from tclahr/fix_azure_sas_storage_url
Fix azure sas storage url
2 parents c805cf2 + 0b75c55 commit fd0efa1

9 files changed

+75
-75
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
66

77
### Highlights
88

9+
- The output and log file names are now automatically appended to the URL provided in `--azure-storage-sas-url` ([#389](https://github.com/tclahr/uac/issues/389)). Consequently, the `--azure-storage-sas-url-log-file` option is no longer needed and has been removed.
910
- Introduced the `statf` tool, which leverages the `stat` system call to produce file status information in body file format for FreeBSD-based systems tlacking the `stat` and `perl` tools.
1011

1112
### Artifacts

lib/aws_s3_presigned_url_transfer.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
# false on fail
1212
_aws_s3_presigned_url_transfer()
1313
{
14-
__aw_payload="${1:-Testing S3 upload from shell script.}"
14+
__aw_payload="${1:-Testing upload from shell script.}"
1515
__aw_url="${2:-}"
1616

1717
__aw_date=`date "+%a, %d %b %Y %H:%M:%S %z"`

lib/azure_storage_sas_url_transfer.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
# false on fail
1212
_azure_storage_sas_url_transfer()
1313
{
14-
__au_payload="${1:-Testing S3 upload from shell script.}"
14+
__au_payload="${1:-Testing upload from shell script.}"
1515
__au_url="${2:-}"
1616

1717
__au_date=`date "+%a, %d %b %Y %H:%M:%S %z"`

lib/parse_command_line_arguments.sh

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -340,15 +340,6 @@ _parse_command_line_arguments()
340340
return 1
341341
fi
342342
;;
343-
"--azure-storage-sas-url-log-file")
344-
if [ -n "${2:-}" ]; then
345-
__UAC_AZURE_STORAGE_SAS_URL_LOG_FILE="${2}"
346-
shift
347-
else
348-
_error_msg "Option '${1}' requires an argument.\nTry 'uac --help' for more information."
349-
return 1
350-
fi
351-
;;
352343
"--delete-local-on-successful-transfer")
353344
__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER=true
354345
;;

lib/s3_transfer_amazon.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# false on fail
1515
_s3_transfer_amazon()
1616
{
17-
__s3a_payload="${1:-Testing S3 upload from shell script.}"
17+
__s3a_payload="${1:-Testing upload from shell script.}"
1818
__s3a_region="${2:-us-east-1}"
1919
__s3a_bucket="${3:-}"
2020
__s3a_access_key="${4:-}"

lib/s3_transfer_google.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# false on fail
1313
_s3_transfer_google()
1414
{
15-
__s3g_payload="${1:-Testing S3 upload from shell script.}"
15+
__s3g_payload="${1:-Testing upload from shell script.}"
1616
__s3g_bucket="${2:-}"
1717
__s3g_token="${3:-}"
1818

lib/s3_transfer_ibm.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# false on fail
1414
_s3_transfer_ibm()
1515
{
16-
__s3i_payload="${1:-Testing S3 upload from shell script.}"
16+
__s3i_payload="${1:-Testing upload from shell script.}"
1717
__s3i_region="${2:-us-south}"
1818
__s3i_bucket="${3:-}"
1919
__s3i_token="${4:-}"

lib/usage.sh

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ Informational Arguments:
9494
9595
Remote Transfer Arguments:
9696
--sftp SERVER
97-
Transfer the output file to remote SFTP server.
97+
Transfer the output and log file to remote SFTP server.
9898
SERVER must be specified in the form [user@]host:[path]
9999
--sftp-port PORT
100100
Remote SFTP server port (default: 22).
@@ -105,7 +105,7 @@ Remote Transfer Arguments:
105105
Allow setting SSH options as key=value pairs.
106106
Can be used multiple times to set multiple options.
107107
--s3-provider
108-
Transfer the output and log files to S3 service.
108+
Transfer the output and log file to S3 service.
109109
Options: amazon, google, ibm
110110
--s3-region
111111
S3 region name (default: us-east-1 [amazon], us-south [ibm]).
@@ -124,10 +124,7 @@ Remote Transfer Arguments:
124124
Transfer the log file to AWS S3 using a pre-signed URL.
125125
Use single quotes to enclose the URL.
126126
--azure-storage-sas-url URL
127-
Transfer the output file to Azure Storage using a SAS URL.
128-
Use single quotes to enclose the URL.
129-
--azure-storage-sas-url-log-file URL
130-
Transfer the log file to Azure Storage using a SAS URL.
127+
Transfer the output and log file to Azure Storage using a SAS URL.
131128
Use single quotes to enclose the URL.
132129
--delete-local-on-successful-transfer
133130
Delete local output and log files on successful transfer.

uac

Lines changed: 66 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,6 @@ __UAC_S3_TOKEN=""
9494
__UAC_AWS_S3_PRESIGNED_URL=""
9595
__UAC_AWS_S3_PRESIGNED_URL_LOG_FILE=""
9696
__UAC_AZURE_STORAGE_SAS_URL=""
97-
__UAC_AZURE_STORAGE_SAS_URL_LOG_FILE=""
9897
__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER=false
9998
__UAC_DESTINATION_DIR=""
10099
__UAC_USER_HOME_LIST=""
@@ -268,31 +267,38 @@ if [ -n "${__UAC_AWS_S3_PRESIGNED_URL}" ]; then
268267
if [ "${__UAC_OUTPUT_FORMAT}" = "none" ]; then
269268
_exit_fatal "AWS S3 transfer aborted: Output format is set to 'none'."
270269
fi
271-
__UAC_AWS_S3_PRESIGNED_URL=`echo "${__UAC_AWS_S3_PRESIGNED_URL}" \
272-
| sed -e "s|%hostname%|${__UAC_HOSTNAME}|g" \
273-
-e "s|%os%|${__UAC_OPERATING_SYSTEM}|g" \
274-
-e "s|%timestamp%|${__ua_current_date_time}|g"`
275-
printf "Validating connectivity to the AWS S3 presigned URL...\n"
276-
_aws_s3_presigned_url_transfer \
277-
"" \
278-
"${__UAC_AWS_S3_PRESIGNED_URL}" \
279-
|| _exit_fatal "Failed to connect to the AWS S3 presigned URL."
270+
if command_exists "curl" || command_exists "wget"; then
271+
__UAC_AWS_S3_PRESIGNED_URL=`echo "${__UAC_AWS_S3_PRESIGNED_URL}" \
272+
| sed -e "s|%hostname%|${__UAC_HOSTNAME}|g" \
273+
-e "s|%os%|${__UAC_OPERATING_SYSTEM}|g" \
274+
-e "s|%timestamp%|${__ua_current_date_time}|g"`
275+
printf "Validating connectivity to the AWS S3 presigned URL...\n"
276+
_aws_s3_presigned_url_transfer \
277+
"" \
278+
"${__UAC_AWS_S3_PRESIGNED_URL}" \
279+
|| _exit_fatal "Failed to connect to the AWS S3 presigned URL."
280+
else
281+
_exit_fatal "Cannot transfer to AWS S3 presigned URL: neither 'curl' nor 'wget' is available."
282+
fi
280283
fi
281284

282285
# test the connectivity to Azure Storage SAS URL
283286
if [ -n "${__UAC_AZURE_STORAGE_SAS_URL}" ]; then
284287
if [ "${__UAC_OUTPUT_FORMAT}" = "none" ]; then
285288
_exit_fatal "Azure Storage SAS URL transfer aborted: Output format is set to 'none'."
286289
fi
287-
__UAC_AZURE_STORAGE_SAS_URL=`echo "${__UAC_AZURE_STORAGE_SAS_URL}" \
288-
| sed -e "s|%hostname%|${__UAC_HOSTNAME}|g" \
289-
-e "s|%os%|${__UAC_OPERATING_SYSTEM}|g" \
290-
-e "s|%timestamp%|${__ua_current_date_time}|g"`
291-
printf "Validating connectivity to the Azure Storage SAS URL...\n"
292-
_azure_storage_sas_url_transfer \
293-
"" \
294-
"${__UAC_AZURE_STORAGE_SAS_URL}" \
295-
|| _exit_fatal "Failed to connect to the Azure Storage SAS URL."
290+
if command_exists "curl" || command_exists "wget"; then
291+
# add output file name to Azure Storage SAS URL
292+
__UAC_AZURE_STORAGE_SAS_URL_TEMP=`echo "${__UAC_AZURE_STORAGE_SAS_URL}" | sed -e "s|?|/transfer_test_from_uac.txt?|"`
293+
294+
printf "Validating connectivity to the Azure Storage SAS URL...\n"
295+
_azure_storage_sas_url_transfer \
296+
"" \
297+
"${__UAC_AZURE_STORAGE_SAS_URL_TEMP}" \
298+
|| _exit_fatal "Failed to connect to the Azure Storage SAS URL."
299+
else
300+
_exit_fatal "Cannot transfer to Azure Storage SAS URL: neither 'curl' nor 'wget' is available."
301+
fi
296302
fi
297303

298304
# check whether temp-dir exists
@@ -660,15 +666,15 @@ if [ -n "${__UAC_S3_PROVIDER}" ] && [ -n "${__UAC_S3_BUCKET}" ]; then
660666
case "${__UAC_S3_PROVIDER}" in
661667
"amazon")
662668
printf "Transferring '%s' to S3 server...\n" \
663-
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
669+
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
664670
if _s3_transfer_amazon \
665-
"${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}" \
671+
"${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}" \
666672
"${__UAC_S3_REGION}" \
667673
"${__UAC_S3_BUCKET}" \
668674
"${__UAC_S3_ACCESS_KEY}" \
669675
"${__UAC_S3_SECRET_KEY}"; then
670676
printf "Transfer completed successfully.\n"
671-
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
677+
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
672678
else
673679
_error_msg "Failed to transfer output file to the S3 server."
674680
fi
@@ -687,13 +693,13 @@ if [ -n "${__UAC_S3_PROVIDER}" ] && [ -n "${__UAC_S3_BUCKET}" ]; then
687693
;;
688694
"google")
689695
printf "Transferring '%s' to S3 server...\n" \
690-
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
696+
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
691697
if _s3_transfer_google \
692-
"${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}" \
698+
"${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}" \
693699
"${__UAC_S3_BUCKET}" \
694700
"${__UAC_S3_TOKEN}"; then
695701
printf "Transfer completed successfully.\n"
696-
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
702+
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
697703
else
698704
_error_msg "Failed to transfer output file to the S3 server."
699705
fi
@@ -710,14 +716,14 @@ if [ -n "${__UAC_S3_PROVIDER}" ] && [ -n "${__UAC_S3_BUCKET}" ]; then
710716
;;
711717
"ibm")
712718
printf "Transferring '%s' to S3 server...\n" \
713-
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
719+
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
714720
if _s3_transfer_ibm \
715-
"${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}" \
721+
"${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}" \
716722
"${__UAC_S3_REGION}" \
717723
"${__UAC_S3_BUCKET}" \
718724
"${__UAC_S3_TOKEN}"; then
719725
printf "Transfer completed successfully.\n"
720-
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
726+
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
721727
else
722728
_error_msg "Failed to transfer output file to the S3 server."
723729
fi
@@ -741,60 +747,65 @@ fi
741747
if [ -n "${__UAC_AWS_S3_PRESIGNED_URL}" ]; then
742748
cd "${__UAC_DESTINATION_DIR}" || _exit_fatal "cd: ${__UAC_DESTINATION_DIR}: No such file or directory"
743749
printf "Transferring '%s' to Amazon S3 presigned URL...\n" \
744-
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
750+
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
745751
if _aws_s3_presigned_url_transfer \
746-
"${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}" \
752+
"${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}" \
747753
"${__UAC_AWS_S3_PRESIGNED_URL}"; then
748754
printf "Transfer completed successfully.\n"
749-
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
755+
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
750756
else
751757
_error_msg "Failed to transfer output file to the S3 server."
752758
fi
753759
if [ -n "${__UAC_AWS_S3_PRESIGNED_URL_LOG_FILE}" ]; then
754-
__UAC_AWS_S3_PRESIGNED_URL_LOG_FILE=`echo "${__UAC_AWS_S3_PRESIGNED_URL_LOG_FILE}" \
755-
| sed -e "s|%hostname%|${__UAC_HOSTNAME}|g" \
756-
-e "s|%os%|${__UAC_OPERATING_SYSTEM}|g" \
757-
-e "s|%timestamp%|${__ua_current_date_time}|g"`
758760
printf "Transferring '%s' to Amazon S3 presigned URL...\n" "${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}.log"
759761
if _aws_s3_presigned_url_transfer \
760762
"${__UAC_OUTPUT_BASE_NAME}.log" \
761763
"${__UAC_AWS_S3_PRESIGNED_URL_LOG_FILE}"; then
762764
printf "Transfer completed successfully.\n"
763-
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
765+
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}.log"
764766
else
765-
_error_msg "Failed to transfer output file to the S3 server."
767+
_error_msg "Failed to transfer log file to the S3 server."
766768
fi
767769
fi
768770
cd "${__UAC_DIR}" || _exit_fatal "cd: ${__UAC_DIR}: No such file or directory"
769771
fi
770772

771773
# transfer output and log file to Azure Storage SAS URL
772774
if [ -n "${__UAC_AZURE_STORAGE_SAS_URL}" ]; then
775+
# add output file name to Azure Storage SAS URL
776+
__UAC_AZURE_STORAGE_SAS_URL_TEMP=`echo "${__UAC_AZURE_STORAGE_SAS_URL}" | sed -e "s|?|/${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}?|"`
777+
778+
__UAC_AZURE_STORAGE_SAS_URL_TEMP=`echo "${__UAC_AZURE_STORAGE_SAS_URL_TEMP}" \
779+
| sed -e "s|%hostname%|${__UAC_HOSTNAME}|g" \
780+
-e "s|%os%|${__UAC_OPERATING_SYSTEM}|g" \
781+
-e "s|%timestamp%|${__ua_current_date_time}|g"`
782+
773783
cd "${__UAC_DESTINATION_DIR}" || _exit_fatal "cd: ${__UAC_DESTINATION_DIR}: No such file or directory"
774784
printf "Transferring '%s' to Azure Storage SAS URL...\n" \
775-
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
785+
"${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
776786
if _azure_storage_sas_url_transfer \
777-
"${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}" \
778-
"${__UAC_AZURE_STORAGE_SAS_URL}";then
787+
"${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}" \
788+
"${__UAC_AZURE_STORAGE_SAS_URL_TEMP}";then
779789
printf "Transfer completed successfully.\n"
780-
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}${__UAC_OUTPUT_EXTENSION:+.}${__UAC_OUTPUT_EXTENSION}"
790+
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}.${__UAC_OUTPUT_EXTENSION}"
781791
else
782792
_error_msg "Failed to transfer output file to Azure Storage SAS URL."
783793
fi
784-
if [ -n "${__UAC_AZURE_STORAGE_SAS_URL_LOG_FILE}" ]; then
785-
__UAC_AZURE_STORAGE_SAS_URL_LOG_FILE=`echo "${__UAC_AZURE_STORAGE_SAS_URL_LOG_FILE}" \
786-
| sed -e "s|%hostname%|${__UAC_HOSTNAME}|g" \
787-
-e "s|%os%|${__UAC_OPERATING_SYSTEM}|g" \
788-
-e "s|%timestamp%|${__ua_current_date_time}|g"`
789-
printf "Transferring '%s' to Azure Storage SAS URL...\n" "${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}.log"
790-
if _azure_storage_sas_url_transfer \
791-
"${__UAC_OUTPUT_BASE_NAME}.log" \
792-
"${__UAC_AZURE_STORAGE_SAS_URL_LOG_FILE}"; then
793-
printf "Transfer completed successfully.\n"
794-
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}.log"
795-
else
796-
_error_msg "Failed to transfer output file to Azure Storage SAS URL."
797-
fi
794+
# add output file name to Azure Storage SAS URL
795+
__UAC_AZURE_STORAGE_SAS_URL_TEMP=`echo "${__UAC_AZURE_STORAGE_SAS_URL}" | sed -e "s|?|/${__UAC_OUTPUT_BASE_NAME}.log?|"`
796+
797+
__UAC_AZURE_STORAGE_SAS_URL_TEMP=`echo "${__UAC_AZURE_STORAGE_SAS_URL_TEMP}" \
798+
| sed -e "s|%hostname%|${__UAC_HOSTNAME}|g" \
799+
-e "s|%os%|${__UAC_OPERATING_SYSTEM}|g" \
800+
-e "s|%timestamp%|${__ua_current_date_time}|g"`
801+
printf "Transferring '%s' to Azure Storage SAS URL...\n" "${__UAC_DESTINATION_DIR}/${__UAC_OUTPUT_BASE_NAME}.log"
802+
if _azure_storage_sas_url_transfer \
803+
"${__UAC_OUTPUT_BASE_NAME}.log" \
804+
"${__UAC_AZURE_STORAGE_SAS_URL_TEMP}"; then
805+
printf "Transfer completed successfully.\n"
806+
${__UAC_DELETE_LOCAL_ON_SUCCESSFUL_TRANSFER} && rm -rf "${__UAC_OUTPUT_BASE_NAME}.log"
807+
else
808+
_error_msg "Failed to transfer log file to Azure Storage SAS URL."
798809
fi
799810
cd "${__UAC_DIR}" || _exit_fatal "cd: ${__UAC_DIR}: No such file or directory"
800811
fi

0 commit comments

Comments
 (0)