You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

wget-to-ipfs.sh 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397
  1. #!/usr/bin/env bash
  2. # This file:
  3. #
  4. # - Demos BASH3 Boilerplate (change this for your script)
  5. #
  6. # Usage:
  7. #
  8. # LOG_LEVEL=7 ./main.sh -f /tmp/x -d (change this for your script)
  9. #
  10. # Based on a template by BASH3 Boilerplate v2.3.0
  11. # http://bash3boilerplate.sh/#authors
  12. #
  13. # The MIT License (MIT)
  14. # Copyright (c) 2013 Kevin van Zonneveld and contributors
  15. # You are not obligated to bundle the LICENSE file with your b3bp projects as long
  16. # as you leave these references intact in the header comments of your source files.
  17. # Exit on error. Append "|| true" if you expect an error.
  18. set -o errexit
  19. # Exit on error inside any functions or subshells.
  20. set -o errtrace
  21. # Do not allow use of undefined vars. Use ${VAR:-} to use an undefined VAR
  22. set -o nounset
  23. # Catch the error in case mysqldump fails (but gzip succeeds) in `mysqldump |gzip`
  24. set -o pipefail
  25. # Turn on traces, useful while debugging but commented out by default
  26. # set -o xtrace
  27. if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
  28. __i_am_main_script="0" # false
  29. if [[ "${__usage+x}" ]]; then
  30. if [[ "${BASH_SOURCE[1]}" = "${0}" ]]; then
  31. __i_am_main_script="1" # true
  32. fi
  33. __b3bp_external_usage="true"
  34. __b3bp_tmp_source_idx=1
  35. fi
  36. else
  37. __i_am_main_script="1" # true
  38. [[ "${__usage+x}" ]] && unset -v __usage
  39. [[ "${__helptext+x}" ]] && unset -v __helptext
  40. fi
  41. # Set magic variables for current file, directory, os, etc.
  42. __dir="$(cd "$(dirname "${BASH_SOURCE[${__b3bp_tmp_source_idx:-0}]}")" && pwd)"
  43. __file="${__dir}/$(basename "${BASH_SOURCE[${__b3bp_tmp_source_idx:-0}]}")"
  44. __base="$(basename "${__file}" .sh)"
  45. # Define the environment variables (and their defaults) that this script depends on
  46. LOG_LEVEL="${LOG_LEVEL:-6}" # 7 = debug -> 0 = emergency
  47. NO_COLOR="${NO_COLOR:-}" # true = disable color. otherwise autodetected
  48. ### Functions
  49. ##############################################################################
  50. function __b3bp_log () {
  51. local log_level="${1}"
  52. shift
  53. # shellcheck disable=SC2034
  54. local color_debug="\x1b[35m"
  55. # shellcheck disable=SC2034
  56. local color_info="\x1b[32m"
  57. # shellcheck disable=SC2034
  58. local color_notice="\x1b[34m"
  59. # shellcheck disable=SC2034
  60. local color_warning="\x1b[33m"
  61. # shellcheck disable=SC2034
  62. local color_error="\x1b[31m"
  63. # shellcheck disable=SC2034
  64. local color_critical="\x1b[1;31m"
  65. # shellcheck disable=SC2034
  66. local color_alert="\x1b[1;33;41m"
  67. # shellcheck disable=SC2034
  68. local color_emergency="\x1b[1;4;5;33;41m"
  69. local colorvar="color_${log_level}"
  70. local color="${!colorvar:-${color_error}}"
  71. local color_reset="\x1b[0m"
  72. if [[ "${NO_COLOR:-}" = "true" ]] || ( [[ "${TERM:-}" != "xterm"* ]] && [[ "${TERM:-}" != "screen"* ]] ) || [[ ! -t 2 ]]; then
  73. if [[ "${NO_COLOR:-}" != "false" ]]; then
  74. # Don't use colors on pipes or non-recognized terminals
  75. color=""; color_reset=""
  76. fi
  77. fi
  78. # all remaining arguments are to be printed
  79. local log_line=""
  80. while IFS=$'\n' read -r log_line; do
  81. echo -e "$(date -u +"%Y-%m-%d %H:%M:%S UTC") ${color}$(printf "[%9s]" "${log_level}")${color_reset} ${log_line}" 1>&2
  82. done <<< "${@:-}"
  83. }
  84. function emergency () { __b3bp_log emergency "${@}"; exit 1; }
  85. function alert () { [[ "${LOG_LEVEL:-0}" -ge 1 ]] && __b3bp_log alert "${@}"; true; }
  86. function critical () { [[ "${LOG_LEVEL:-0}" -ge 2 ]] && __b3bp_log critical "${@}"; true; }
  87. function error () { [[ "${LOG_LEVEL:-0}" -ge 3 ]] && __b3bp_log error "${@}"; true; }
  88. function warning () { [[ "${LOG_LEVEL:-0}" -ge 4 ]] && __b3bp_log warning "${@}"; true; }
  89. function notice () { [[ "${LOG_LEVEL:-0}" -ge 5 ]] && __b3bp_log notice "${@}"; true; }
  90. function info () { [[ "${LOG_LEVEL:-0}" -ge 6 ]] && __b3bp_log info "${@}"; true; }
  91. function debug () { [[ "${LOG_LEVEL:-0}" -ge 7 ]] && __b3bp_log debug "${@}"; true; }
  92. function help () {
  93. echo "" 1>&2
  94. echo " ${*}" 1>&2
  95. echo "" 1>&2
  96. echo " ${__usage:-No usage available}" 1>&2
  97. echo "" 1>&2
  98. if [[ "${__helptext:-}" ]]; then
  99. echo " ${__helptext}" 1>&2
  100. echo "" 1>&2
  101. fi
  102. exit 1
  103. }
  104. function _check_binary() {
  105. if [[ "$(which ${@})" = "" ]]; then
  106. error "Command '${@}' not found. Ensure ${@} is installed and in the PATH"
  107. exit 1
  108. fi
  109. }
  110. function date_iso_8601() {
  111. date -u +%FT%T%z
  112. }
  113. ### Parse commandline options
  114. ##############################################################################
  115. # Commandline options. This defines the usage page, and is used to parse cli
  116. # opts & defaults from. The parsing is unforgiving so be precise in your syntax
  117. # - A short option must be preset for every long option; but every short option
  118. # need not have a long option
  119. # - `--` is respected as the separator between options and arguments
  120. # - We do not bash-expand defaults, so setting '~/app' as a default will not resolve to ${HOME}.
  121. # you can use bash variables to work around this (so use ${HOME} instead)
  122. # shellcheck disable=SC2015
  123. [[ "${__usage+x}" ]] || read -r -d '' __usage <<-'EOF' || true # exits non-zero when EOF encountered
  124. -u --url [arg] URL to download. Required.
  125. -r --root [arg] IPFS id of a directory to add the file to. Leave empty when you don't want to add the file to a directory.
  126. -p --pinbook [arg] Location of the optional Pinfile where pinned hashes are stored.
  127. -v Enable verbose mode, print script as it is executed
  128. -d --debug Enables debug mode
  129. -h --help This page
  130. -n --no-color Disable color output
  131. EOF
  132. # shellcheck disable=SC2015
  133. [[ "${__helptext+x}" ]] || read -r -d '' __helptext <<-'EOF' || true # exits non-zero when EOF encountered
  134. Downloads a file from an URL and adds to IPFS.
  135. EOF
  136. # Translate usage string -> getopts arguments, and set $arg_<flag> defaults
  137. while read -r __b3bp_tmp_line; do
  138. if [[ "${__b3bp_tmp_line}" =~ ^- ]]; then
  139. # fetch single character version of option string
  140. __b3bp_tmp_opt="${__b3bp_tmp_line%% *}"
  141. __b3bp_tmp_opt="${__b3bp_tmp_opt:1}"
  142. # fetch long version if present
  143. __b3bp_tmp_long_opt=""
  144. if [[ "${__b3bp_tmp_line}" = *"--"* ]]; then
  145. __b3bp_tmp_long_opt="${__b3bp_tmp_line#*--}"
  146. __b3bp_tmp_long_opt="${__b3bp_tmp_long_opt%% *}"
  147. fi
  148. # map opt long name to+from opt short name
  149. printf -v "__b3bp_tmp_opt_long2short_${__b3bp_tmp_long_opt//-/_}" '%s' "${__b3bp_tmp_opt}"
  150. printf -v "__b3bp_tmp_opt_short2long_${__b3bp_tmp_opt}" '%s' "${__b3bp_tmp_long_opt//-/_}"
  151. # check if option takes an argument
  152. if [[ "${__b3bp_tmp_line}" =~ \[.*\] ]]; then
  153. __b3bp_tmp_opt="${__b3bp_tmp_opt}:" # add : if opt has arg
  154. __b3bp_tmp_init="" # it has an arg. init with ""
  155. printf -v "__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}" '%s' "1"
  156. elif [[ "${__b3bp_tmp_line}" =~ \{.*\} ]]; then
  157. __b3bp_tmp_opt="${__b3bp_tmp_opt}:" # add : if opt has arg
  158. __b3bp_tmp_init="" # it has an arg. init with ""
  159. # remember that this option requires an argument
  160. printf -v "__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}" '%s' "2"
  161. else
  162. __b3bp_tmp_init="0" # it's a flag. init with 0
  163. printf -v "__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}" '%s' "0"
  164. fi
  165. __b3bp_tmp_opts="${__b3bp_tmp_opts:-}${__b3bp_tmp_opt}"
  166. fi
  167. [[ "${__b3bp_tmp_opt:-}" ]] || continue
  168. if [[ "${__b3bp_tmp_line}" =~ (^|\.\ *)Default= ]]; then
  169. # ignore default value if option does not have an argument
  170. __b3bp_tmp_varname="__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}"
  171. if [[ "${!__b3bp_tmp_varname}" != "0" ]]; then
  172. __b3bp_tmp_init="${__b3bp_tmp_line##*Default=}"
  173. __b3bp_tmp_re='^"(.*)"$'
  174. if [[ "${__b3bp_tmp_init}" =~ ${__b3bp_tmp_re} ]]; then
  175. __b3bp_tmp_init="${BASH_REMATCH[1]}"
  176. else
  177. __b3bp_tmp_re="^'(.*)'$"
  178. if [[ "${__b3bp_tmp_init}" =~ ${__b3bp_tmp_re} ]]; then
  179. __b3bp_tmp_init="${BASH_REMATCH[1]}"
  180. fi
  181. fi
  182. fi
  183. fi
  184. if [[ "${__b3bp_tmp_line}" =~ (^|\.\ *)Required\. ]]; then
  185. # remember that this option requires an argument
  186. printf -v "__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}" '%s' "2"
  187. fi
  188. printf -v "arg_${__b3bp_tmp_opt:0:1}" '%s' "${__b3bp_tmp_init}"
  189. done <<< "${__usage:-}"
  190. # run getopts only if options were specified in __usage
  191. if [[ "${__b3bp_tmp_opts:-}" ]]; then
  192. # Allow long options like --this
  193. __b3bp_tmp_opts="${__b3bp_tmp_opts}-:"
  194. # Reset in case getopts has been used previously in the shell.
  195. OPTIND=1
  196. # start parsing command line
  197. set +o nounset # unexpected arguments will cause unbound variables
  198. # to be dereferenced
  199. # Overwrite $arg_<flag> defaults with the actual CLI options
  200. while getopts "${__b3bp_tmp_opts}" __b3bp_tmp_opt; do
  201. [[ "${__b3bp_tmp_opt}" = "?" ]] && help "Invalid use of script: ${*} "
  202. if [[ "${__b3bp_tmp_opt}" = "-" ]]; then
  203. # OPTARG is long-option-name or long-option=value
  204. if [[ "${OPTARG}" =~ .*=.* ]]; then
  205. # --key=value format
  206. __b3bp_tmp_long_opt=${OPTARG/=*/}
  207. # Set opt to the short option corresponding to the long option
  208. __b3bp_tmp_varname="__b3bp_tmp_opt_long2short_${__b3bp_tmp_long_opt//-/_}"
  209. printf -v "__b3bp_tmp_opt" '%s' "${!__b3bp_tmp_varname}"
  210. OPTARG=${OPTARG#*=}
  211. else
  212. # --key value format
  213. # Map long name to short version of option
  214. __b3bp_tmp_varname="__b3bp_tmp_opt_long2short_${OPTARG//-/_}"
  215. printf -v "__b3bp_tmp_opt" '%s' "${!__b3bp_tmp_varname}"
  216. # Only assign OPTARG if option takes an argument
  217. __b3bp_tmp_varname="__b3bp_tmp_has_arg_${__b3bp_tmp_opt}"
  218. printf -v "OPTARG" '%s' "${@:OPTIND:${!__b3bp_tmp_varname}}"
  219. # shift over the argument if argument is expected
  220. ((OPTIND+=__b3bp_tmp_has_arg_${__b3bp_tmp_opt}))
  221. fi
  222. # we have set opt/OPTARG to the short value and the argument as OPTARG if it exists
  223. fi
  224. __b3bp_tmp_varname="arg_${__b3bp_tmp_opt:0:1}"
  225. __b3bp_tmp_default="${!__b3bp_tmp_varname}"
  226. __b3bp_tmp_value="${OPTARG}"
  227. if [[ -z "${OPTARG}" ]] && [[ "${__b3bp_tmp_default}" = "0" ]]; then
  228. __b3bp_tmp_value="1"
  229. fi
  230. printf -v "${__b3bp_tmp_varname}" '%s' "${__b3bp_tmp_value}"
  231. debug "cli arg ${__b3bp_tmp_varname} = (${__b3bp_tmp_default}) -> ${!__b3bp_tmp_varname}"
  232. done
  233. set -o nounset # no more unbound variable references expected
  234. shift $((OPTIND-1))
  235. if [[ "${1:-}" = "--" ]] ; then
  236. shift
  237. fi
  238. fi
  239. ### Automatic validation of required option arguments
  240. ##############################################################################
  241. for __b3bp_tmp_varname in ${!__b3bp_tmp_has_arg_*}; do
  242. # validate only options which required an argument
  243. [[ "${!__b3bp_tmp_varname}" = "2" ]] || continue
  244. __b3bp_tmp_opt_short="${__b3bp_tmp_varname##*_}"
  245. __b3bp_tmp_varname="arg_${__b3bp_tmp_opt_short}"
  246. [[ "${!__b3bp_tmp_varname}" ]] && continue
  247. __b3bp_tmp_varname="__b3bp_tmp_opt_short2long_${__b3bp_tmp_opt_short}"
  248. printf -v "__b3bp_tmp_opt_long" '%s' "${!__b3bp_tmp_varname}"
  249. [[ "${__b3bp_tmp_opt_long:-}" ]] && __b3bp_tmp_opt_long=" (--${__b3bp_tmp_opt_long//_/-})"
  250. help "Option -${__b3bp_tmp_opt_short}${__b3bp_tmp_opt_long:-} requires an argument"
  251. done
  252. ### Cleanup Environment variables
  253. ##############################################################################
  254. for __tmp_varname in ${!__b3bp_tmp_*}; do
  255. unset -v "${__tmp_varname}"
  256. done
  257. unset -v __tmp_varname
  258. ### Externally supplied __usage. Nothing else to do here
  259. ##############################################################################
  260. if [[ "${__b3bp_external_usage:-}" = "true" ]]; then
  261. unset -v __b3bp_external_usage
  262. return
  263. fi
  264. ### Signal trapping and backtracing
  265. ##############################################################################
  266. function __b3bp_cleanup_before_exit () {
  267. info "Cleaning up. Done"
  268. }
  269. trap __b3bp_cleanup_before_exit EXIT
  270. # requires `set -o errtrace`
  271. __b3bp_err_report() {
  272. local error_code
  273. error_code=${?}
  274. error "Error in ${__file} in function ${1} on line ${2}"
  275. exit ${error_code}
  276. }
  277. # Uncomment the following line for always providing an error backtrace
  278. # trap '__b3bp_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
  279. ### Command-line argument switches (like -d for debugmode, -h for showing helppage)
  280. ##############################################################################
  281. # debug mode
  282. if [[ "${arg_d:?}" = "1" ]]; then
  283. set -o xtrace
  284. LOG_LEVEL="7"
  285. # Enable error backtracing
  286. trap '__b3bp_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
  287. fi
  288. # verbose mode
  289. if [[ "${arg_v:?}" = "1" ]]; then
  290. set -o verbose
  291. fi
  292. # no color mode
  293. if [[ "${arg_n:?}" = "1" ]]; then
  294. NO_COLOR="true"
  295. fi
  296. # help mode
  297. if [[ "${arg_h:?}" = "1" ]]; then
  298. # Help exists with code 1
  299. help "Help using ${0}"
  300. fi
  301. ### Validation. Error out if the things required for your script are not present
  302. ##############################################################################
  303. [[ "${arg_u:-}" ]] || help "Setting an URL with -u or --url is required"
  304. [[ "${LOG_LEVEL:-}" ]] || emergency "Cannot continue without LOG_LEVEL. "
  305. ### Runtime
  306. ##############################################################################
  307. _check_binary "ipfs"
  308. _file_name="$(basename "${arg_u}")"
  309. _file_hash="$(ipfs add --nocopy --cid-version=1 --quieter ${arg_u})"
  310. info "Added ${arg_u} as ${_file_hash}"
  311. if [[ -n ${arg_p} ]]; then
  312. echo "$(date_iso_8601) ${_file_hash} ${arg_u}" >> ${arg_p}
  313. fi
  314. if [[ -n ${arg_r} ]]; then
  315. _new_dir="$(ipfs object patch ${arg_r} add-link $_file_name $_file_hash)"
  316. info "Placed ${arg_u} in directory, new root is ${_new_dir}"
  317. fi
  318. echo "file: ${_file_hash}"
  319. echo "root: ${_new_dir-'no root provided'}"