#!/usr/bin/env bash # # send_dir.sh — upload every file in a directory to a running FileServer. # # Each file is POSTed to /upload as multipart/form-data. By default one file per # request (so you get per-file feedback); use -b to batch all files into a # single request instead. # # Usage: # bin/send_dir.sh [-u URL] [-b] [-r] DIR # # DIR directory whose files are uploaded # -u URL server base URL (default: http://127.0.0.1:8080, or $FS_URL) # -b batch: send all files in one multipart request # -r recurse into subdirectories # -j N upload up to N files concurrently (default: 1). A slow big # file no longer blocks the others; results are tallied at the end. # # Examples: # bin/send_dir.sh data/samples # bin/send_dir.sh -u http://host:9000 -b -r ~/Downloads set -euo pipefail url="${FS_URL:-http://127.0.0.1:8080}" batch=0 recurse=0 jobs=1 while getopts ":u:brj:h" opt; do case "$opt" in u) url="$OPTARG" ;; b) batch=1 ;; r) recurse=1 ;; j) jobs="$OPTARG" ;; h) sed -n '2,22p' "$0"; exit 0 ;; \?) echo "unknown option: -$OPTARG" >&2; exit 2 ;; :) echo "option -$OPTARG needs an argument" >&2; exit 2 ;; esac done shift $((OPTIND - 1)) if ! [[ "$jobs" =~ ^[0-9]+$ ]] || [[ "$jobs" -lt 1 ]]; then echo "-j needs a positive integer, got: $jobs" >&2 exit 2 fi dir="${1:-}" if [[ -z "$dir" ]]; then echo "usage: $0 [-u URL] [-b] [-r] DIR" >&2 exit 2 fi if [[ ! -d "$dir" ]]; then echo "not a directory: $dir" >&2 exit 2 fi endpoint="${url%/}/upload" # Collect regular files (optionally recursive), NUL-safe for odd filenames. mapfile -d '' files < <( if [[ "$recurse" -eq 1 ]]; then find "$dir" -type f -print0 else find "$dir" -maxdepth 1 -type f -print0 fi ) if [[ "${#files[@]}" -eq 0 ]]; then echo "no files found in $dir" >&2 exit 1 fi echo "uploading ${#files[@]} file(s) from '$dir' to $endpoint" if [[ "$batch" -eq 1 ]]; then # One request with every file. Each -F field name is arbitrary; the server # keys off the filename, so we just number them. args=() i=0 for f in "${files[@]}"; do args+=(-F "f${i}=@${f}") i=$((i + 1)) done code=$(curl -sS -o /dev/stderr -w '%{http_code}' "${args[@]}" "$endpoint") echo echo "-> HTTP $code" [[ "$code" == "202" ]] else # One request per file, up to $jobs in flight at once. Each upload runs in the # background so a slow (big) file doesn't stall the ones behind it. Every job # writes its HTTP status to a per-file line in $results; we tally after all # jobs drain. results="$(mktemp)" trap 'rm -f "$results"' EXIT # Upload one file: print progress and append "codepath" to $results. upload_one() { local f="$1" code code=$(curl -sS -o /dev/stderr -w '%{http_code}' -F "file=@${f}" "$endpoint") echo " -> HTTP $code ${f}" printf '%s\t%s\n' "$code" "$f" >> "$results" } for f in "${files[@]}"; do # Throttle: wait for a slot to free up before launching the next upload. # (|| true: a failed upload's exit status must not trip `set -e` here.) while (( $(jobs -r -p | wc -l) >= jobs )); do wait -n || true done upload_one "$f" & done wait || true ok=$(grep -c $'^202\t' "$results" || true) fail=$(( ${#files[@]} - ok )) echo "done: $ok accepted, $fail failed" [[ "$fail" -eq 0 ]] fi