You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

osm-sync.sh 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395
  1. #!/usr/bin/env bash
  2. # Usage:
  3. #
  4. # LOG_LEVEL=7 ./osm-sync.sh
  5. #
  6. # Based on a template by BASH3 Boilerplate v2.3.0
  7. # http://bash3boilerplate.sh/#authors
  8. #
  9. # The MIT License (MIT)
  10. # Copyright (c) 2013 Kevin van Zonneveld and contributors
  11. # You are not obligated to bundle the LICENSE file with your b3bp projects as long
  12. # as you leave these references intact in the header comments of your source files.
  13. # Exit on error. Append "|| true" if you expect an error.
  14. set -o errexit
  15. # Exit on error inside any functions or subshells.
  16. set -o errtrace
  17. # Do not allow use of undefined vars. Use ${VAR:-} to use an undefined VAR
  18. set -o nounset
  19. # Catch the error in case mysqldump fails (but gzip succeeds) in `mysqldump |gzip`
  20. set -o pipefail
  21. # Turn on traces, useful while debugging but commented out by default
  22. # set -o xtrace
  23. if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
  24. __i_am_main_script="0" # false
  25. if [[ "${__usage+x}" ]]; then
  26. if [[ "${BASH_SOURCE[1]}" = "${0}" ]]; then
  27. __i_am_main_script="1" # true
  28. fi
  29. __b3bp_external_usage="true"
  30. __b3bp_tmp_source_idx=1
  31. fi
  32. else
  33. __i_am_main_script="1" # true
  34. [[ "${__usage+x}" ]] && unset -v __usage
  35. [[ "${__helptext+x}" ]] && unset -v __helptext
  36. fi
  37. # Set magic variables for current file, directory, os, etc.
  38. __dir="$(cd "$(dirname "${BASH_SOURCE[${__b3bp_tmp_source_idx:-0}]}")" && pwd)"
  39. __file="${__dir}/$(basename "${BASH_SOURCE[${__b3bp_tmp_source_idx:-0}]}")"
  40. __base="$(basename "${__file}" .sh)"
  41. # Define the environment variables (and their defaults) that this script depends on
  42. LOG_LEVEL="${LOG_LEVEL:-6}" # 7 = debug -> 0 = emergency
  43. NO_COLOR="${NO_COLOR:-}" # true = disable color. otherwise autodetected
  44. ### Functions
  45. ##############################################################################
  46. function __b3bp_log () {
  47. local log_level="${1}"
  48. shift
  49. # shellcheck disable=SC2034
  50. local color_debug="\x1b[35m"
  51. # shellcheck disable=SC2034
  52. local color_info="\x1b[32m"
  53. # shellcheck disable=SC2034
  54. local color_notice="\x1b[34m"
  55. # shellcheck disable=SC2034
  56. local color_warning="\x1b[33m"
  57. # shellcheck disable=SC2034
  58. local color_error="\x1b[31m"
  59. # shellcheck disable=SC2034
  60. local color_critical="\x1b[1;31m"
  61. # shellcheck disable=SC2034
  62. local color_alert="\x1b[1;33;41m"
  63. # shellcheck disable=SC2034
  64. local color_emergency="\x1b[1;4;5;33;41m"
  65. local colorvar="color_${log_level}"
  66. local color="${!colorvar:-${color_error}}"
  67. local color_reset="\x1b[0m"
  68. if [[ "${NO_COLOR:-}" = "true" ]] || ( [[ "${TERM:-}" != "xterm"* ]] && [[ "${TERM:-}" != "screen"* ]] ) || [[ ! -t 2 ]]; then
  69. if [[ "${NO_COLOR:-}" != "false" ]]; then
  70. # Don't use colors on pipes or non-recognized terminals
  71. color=""; color_reset=""
  72. fi
  73. fi
  74. # all remaining arguments are to be printed
  75. local log_line=""
  76. while IFS=$'\n' read -r log_line; do
  77. echo -e "$(date -u +"%Y-%m-%d %H:%M:%S UTC") ${color}$(printf "[%9s]" "${log_level}")${color_reset} ${log_line}" 1>&2
  78. done <<< "${@:-}"
  79. }
  80. function emergency () { __b3bp_log emergency "${@}"; exit 1; }
  81. function alert () { [[ "${LOG_LEVEL:-0}" -ge 1 ]] && __b3bp_log alert "${@}"; true; }
  82. function critical () { [[ "${LOG_LEVEL:-0}" -ge 2 ]] && __b3bp_log critical "${@}"; true; }
  83. function error () { [[ "${LOG_LEVEL:-0}" -ge 3 ]] && __b3bp_log error "${@}"; true; }
  84. function warning () { [[ "${LOG_LEVEL:-0}" -ge 4 ]] && __b3bp_log warning "${@}"; true; }
  85. function notice () { [[ "${LOG_LEVEL:-0}" -ge 5 ]] && __b3bp_log notice "${@}"; true; }
  86. function info () { [[ "${LOG_LEVEL:-0}" -ge 6 ]] && __b3bp_log info "${@}"; true; }
  87. function debug () { [[ "${LOG_LEVEL:-0}" -ge 7 ]] && __b3bp_log debug "${@}"; true; }
  88. function help () {
  89. echo "" 1>&2
  90. echo " ${*}" 1>&2
  91. echo "" 1>&2
  92. echo " ${__usage:-No usage available}" 1>&2
  93. echo "" 1>&2
  94. if [[ "${__helptext:-}" ]]; then
  95. echo " ${__helptext}" 1>&2
  96. echo "" 1>&2
  97. fi
  98. exit 1
  99. }
  100. ### Parse commandline options
  101. ##############################################################################
  102. # Commandline options. This defines the usage page, and is used to parse cli
  103. # opts & defaults from. The parsing is unforgiving so be precise in your syntax
  104. # - A short option must be preset for every long option; but every short option
  105. # need not have a long option
  106. # - `--` is respected as the separator between options and arguments
  107. # - We do not bash-expand defaults, so setting '~/app' as a default will not resolve to ${HOME}.
  108. # you can use bash variables to work around this (so use ${HOME} instead)
  109. # shellcheck disable=SC2015
  110. [[ "${__usage+x}" ]] || read -r -d '' __usage <<-'EOF' || true # exits non-zero when EOF encountered
  111. -u --urls-file [arg] A list of URLS to parse. Space separated. Default="./urls.txt"
  112. -p --pinbook [arg] Pinbook file. Default="./pinbook.txt"
  113. -c --change-dns Update the DNS server over REST.
  114. -v Enable verbose mode, print script as it is executed
  115. -d --debug Enables debug mode
  116. -h --help This page
  117. -n --no-color Disable color output
  118. EOF
  119. # shellcheck disable=SC2015
  120. [[ "${__helptext+x}" ]] || read -r -d '' __helptext <<-'EOF' || true # exits non-zero when EOF encountered
  121. Wraps wget-to-ipfs to recursively add a list of URLs to IPFS.
  122. EOF
  123. # Translate usage string -> getopts arguments, and set $arg_<flag> defaults
  124. while read -r __b3bp_tmp_line; do
  125. if [[ "${__b3bp_tmp_line}" =~ ^- ]]; then
  126. # fetch single character version of option string
  127. __b3bp_tmp_opt="${__b3bp_tmp_line%% *}"
  128. __b3bp_tmp_opt="${__b3bp_tmp_opt:1}"
  129. # fetch long version if present
  130. __b3bp_tmp_long_opt=""
  131. if [[ "${__b3bp_tmp_line}" = *"--"* ]]; then
  132. __b3bp_tmp_long_opt="${__b3bp_tmp_line#*--}"
  133. __b3bp_tmp_long_opt="${__b3bp_tmp_long_opt%% *}"
  134. fi
  135. # map opt long name to+from opt short name
  136. printf -v "__b3bp_tmp_opt_long2short_${__b3bp_tmp_long_opt//-/_}" '%s' "${__b3bp_tmp_opt}"
  137. printf -v "__b3bp_tmp_opt_short2long_${__b3bp_tmp_opt}" '%s' "${__b3bp_tmp_long_opt//-/_}"
  138. # check if option takes an argument
  139. if [[ "${__b3bp_tmp_line}" =~ \[.*\] ]]; then
  140. __b3bp_tmp_opt="${__b3bp_tmp_opt}:" # add : if opt has arg
  141. __b3bp_tmp_init="" # it has an arg. init with ""
  142. printf -v "__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}" '%s' "1"
  143. elif [[ "${__b3bp_tmp_line}" =~ \{.*\} ]]; then
  144. __b3bp_tmp_opt="${__b3bp_tmp_opt}:" # add : if opt has arg
  145. __b3bp_tmp_init="" # it has an arg. init with ""
  146. # remember that this option requires an argument
  147. printf -v "__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}" '%s' "2"
  148. else
  149. __b3bp_tmp_init="0" # it's a flag. init with 0
  150. printf -v "__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}" '%s' "0"
  151. fi
  152. __b3bp_tmp_opts="${__b3bp_tmp_opts:-}${__b3bp_tmp_opt}"
  153. fi
  154. [[ "${__b3bp_tmp_opt:-}" ]] || continue
  155. if [[ "${__b3bp_tmp_line}" =~ (^|\.\ *)Default= ]]; then
  156. # ignore default value if option does not have an argument
  157. __b3bp_tmp_varname="__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}"
  158. if [[ "${!__b3bp_tmp_varname}" != "0" ]]; then
  159. __b3bp_tmp_init="${__b3bp_tmp_line##*Default=}"
  160. __b3bp_tmp_re='^"(.*)"$'
  161. if [[ "${__b3bp_tmp_init}" =~ ${__b3bp_tmp_re} ]]; then
  162. __b3bp_tmp_init="${BASH_REMATCH[1]}"
  163. else
  164. __b3bp_tmp_re="^'(.*)'$"
  165. if [[ "${__b3bp_tmp_init}" =~ ${__b3bp_tmp_re} ]]; then
  166. __b3bp_tmp_init="${BASH_REMATCH[1]}"
  167. fi
  168. fi
  169. fi
  170. fi
  171. if [[ "${__b3bp_tmp_line}" =~ (^|\.\ *)Required\. ]]; then
  172. # remember that this option requires an argument
  173. printf -v "__b3bp_tmp_has_arg_${__b3bp_tmp_opt:0:1}" '%s' "2"
  174. fi
  175. printf -v "arg_${__b3bp_tmp_opt:0:1}" '%s' "${__b3bp_tmp_init}"
  176. done <<< "${__usage:-}"
  177. # run getopts only if options were specified in __usage
  178. if [[ "${__b3bp_tmp_opts:-}" ]]; then
  179. # Allow long options like --this
  180. __b3bp_tmp_opts="${__b3bp_tmp_opts}-:"
  181. # Reset in case getopts has been used previously in the shell.
  182. OPTIND=1
  183. # start parsing command line
  184. set +o nounset # unexpected arguments will cause unbound variables
  185. # to be dereferenced
  186. # Overwrite $arg_<flag> defaults with the actual CLI options
  187. while getopts "${__b3bp_tmp_opts}" __b3bp_tmp_opt; do
  188. [[ "${__b3bp_tmp_opt}" = "?" ]] && help "Invalid use of script: ${*} "
  189. if [[ "${__b3bp_tmp_opt}" = "-" ]]; then
  190. # OPTARG is long-option-name or long-option=value
  191. if [[ "${OPTARG}" =~ .*=.* ]]; then
  192. # --key=value format
  193. __b3bp_tmp_long_opt=${OPTARG/=*/}
  194. # Set opt to the short option corresponding to the long option
  195. __b3bp_tmp_varname="__b3bp_tmp_opt_long2short_${__b3bp_tmp_long_opt//-/_}"
  196. printf -v "__b3bp_tmp_opt" '%s' "${!__b3bp_tmp_varname}"
  197. OPTARG=${OPTARG#*=}
  198. else
  199. # --key value format
  200. # Map long name to short version of option
  201. __b3bp_tmp_varname="__b3bp_tmp_opt_long2short_${OPTARG//-/_}"
  202. printf -v "__b3bp_tmp_opt" '%s' "${!__b3bp_tmp_varname}"
  203. # Only assign OPTARG if option takes an argument
  204. __b3bp_tmp_varname="__b3bp_tmp_has_arg_${__b3bp_tmp_opt}"
  205. printf -v "OPTARG" '%s' "${@:OPTIND:${!__b3bp_tmp_varname}}"
  206. # shift over the argument if argument is expected
  207. ((OPTIND+=__b3bp_tmp_has_arg_${__b3bp_tmp_opt}))
  208. fi
  209. # we have set opt/OPTARG to the short value and the argument as OPTARG if it exists
  210. fi
  211. __b3bp_tmp_varname="arg_${__b3bp_tmp_opt:0:1}"
  212. __b3bp_tmp_default="${!__b3bp_tmp_varname}"
  213. __b3bp_tmp_value="${OPTARG}"
  214. if [[ -z "${OPTARG}" ]] && [[ "${__b3bp_tmp_default}" = "0" ]]; then
  215. __b3bp_tmp_value="1"
  216. fi
  217. printf -v "${__b3bp_tmp_varname}" '%s' "${__b3bp_tmp_value}"
  218. debug "cli arg ${__b3bp_tmp_varname} = (${__b3bp_tmp_default}) -> ${!__b3bp_tmp_varname}"
  219. done
  220. set -o nounset # no more unbound variable references expected
  221. shift $((OPTIND-1))
  222. if [[ "${1:-}" = "--" ]] ; then
  223. shift
  224. fi
  225. fi
  226. ### Automatic validation of required option arguments
  227. ##############################################################################
  228. for __b3bp_tmp_varname in ${!__b3bp_tmp_has_arg_*}; do
  229. # validate only options which required an argument
  230. [[ "${!__b3bp_tmp_varname}" = "2" ]] || continue
  231. __b3bp_tmp_opt_short="${__b3bp_tmp_varname##*_}"
  232. __b3bp_tmp_varname="arg_${__b3bp_tmp_opt_short}"
  233. [[ "${!__b3bp_tmp_varname}" ]] && continue
  234. __b3bp_tmp_varname="__b3bp_tmp_opt_short2long_${__b3bp_tmp_opt_short}"
  235. printf -v "__b3bp_tmp_opt_long" '%s' "${!__b3bp_tmp_varname}"
  236. [[ "${__b3bp_tmp_opt_long:-}" ]] && __b3bp_tmp_opt_long=" (--${__b3bp_tmp_opt_long//_/-})"
  237. help "Option -${__b3bp_tmp_opt_short}${__b3bp_tmp_opt_long:-} requires an argument"
  238. done
  239. ### Cleanup Environment variables
  240. ##############################################################################
  241. for __tmp_varname in ${!__b3bp_tmp_*}; do
  242. unset -v "${__tmp_varname}"
  243. done
  244. unset -v __tmp_varname
  245. ### Externally supplied __usage. Nothing else to do here
  246. ##############################################################################
  247. if [[ "${__b3bp_external_usage:-}" = "true" ]]; then
  248. unset -v __b3bp_external_usage
  249. return
  250. fi
  251. ### Signal trapping and backtracing
  252. ##############################################################################
  253. function __b3bp_cleanup_before_exit () {
  254. info "Cleaning up. Done"
  255. }
  256. trap __b3bp_cleanup_before_exit EXIT
  257. # requires `set -o errtrace`
  258. __b3bp_err_report() {
  259. local error_code
  260. error_code=${?}
  261. error "Error in ${__file} in function ${1} on line ${2}"
  262. exit ${error_code}
  263. }
  264. # Uncomment the following line for always providing an error backtrace
  265. # trap '__b3bp_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
  266. ### Command-line argument switches (like -d for debugmode, -h for showing helppage)
  267. ##############################################################################
  268. # debug mode
  269. if [[ "${arg_d:?}" = "1" ]]; then
  270. set -o xtrace
  271. LOG_LEVEL="7"
  272. # Enable error backtracing
  273. trap '__b3bp_err_report "${FUNCNAME:-.}" ${LINENO}' ERR
  274. fi
  275. # verbose mode
  276. if [[ "${arg_v:?}" = "1" ]]; then
  277. set -o verbose
  278. fi
  279. # no color mode
  280. if [[ "${arg_n:?}" = "1" ]]; then
  281. NO_COLOR="true"
  282. fi
  283. # help mode
  284. if [[ "${arg_h:?}" = "1" ]]; then
  285. # Help exists with code 1
  286. help "Help using ${0}"
  287. fi
  288. ### Validation. Error out if the things required for your script are not present
  289. ##############################################################################
  290. [[ -f "${arg_u}" ]] || help "Cannot read ${arg_u} file"
  291. [[ "${LOG_LEVEL:-}" ]] || emergency "Cannot continue without LOG_LEVEL. "
  292. ### Runtime
  293. ##############################################################################
  294. export _dir_hash=$(ipfs object new unixfs-dir)
  295. ## Remove pins from old files.
  296. if [ ! -f "${arg_p}" ]; then
  297. touch "${arg_p}"
  298. fi
  299. while read -r _line;
  300. do
  301. # lines are "date filehash filename"
  302. _pin=$(echo ${_line} | cut -d " " -f 2)
  303. info $(ipfs pin rm "/ipfs/${_pin}")
  304. done < "${arg_p}"
  305. ## empty old pins
  306. truncate --size=0 "${arg_p}"
  307. ## Add new files
  308. while read -r _url;
  309. do
  310. _out=$(wget-to-ipfs --url="${_url}" --root="${_dir_hash}" --pinbook="${arg_p}")
  311. _dir_hash=$(echo "${_out}" | grep root | cut --delimiter=" " --fields=2)
  312. done < "${arg_u}"
  313. info "new dir: ${_dir_hash}"
  314. ##
  315. # Update the DNSlink
  316. if [[ "${arg_c:?}" = "1" ]]; then
  317. curl -X PUT -d "dnslink=/ipfs/${_dir_hash}" \
  318. --user "${DNS_API_USER}:${DNS_PASSWORD}" \
  319. "https://natalie.berk.es/admin/dns/custom/ipfs.placebazaar.org/txt"
  320. fi