Add -j flag to send_dir.sh for concurrent, non-blocking uploads

bin/send_dir.sh

@@ -13,6 +13,8 @@
 #   -u URL      server base URL (default: http://127.0.0.1:8080, or $FS_URL)
 #   -b          batch: send all files in one multipart request
 #   -r          recurse into subdirectories
+#   -j N        upload up to N files concurrently (default: 1). A slow big
+#               file no longer blocks the others; results are tallied at the end.
 #
 # Examples:
 #   bin/send_dir.sh data/samples
@@ -22,19 +24,26 @@ set -euo pipefail
 url="${FS_URL:-http://127.0.0.1:8080}"
 batch=0
 recurse=0
+jobs=1
 
-while getopts ":u:brh" opt; do
+while getopts ":u:brj:h" opt; do
   case "$opt" in
     u) url="$OPTARG" ;;
     b) batch=1 ;;
     r) recurse=1 ;;
-    h) sed -n '2,20p' "$0"; exit 0 ;;
+    j) jobs="$OPTARG" ;;
+    h) sed -n '2,22p' "$0"; exit 0 ;;
     \?) echo "unknown option: -$OPTARG" >&2; exit 2 ;;
     :) echo "option -$OPTARG needs an argument" >&2; exit 2 ;;
   esac
 done
 shift $((OPTIND - 1))
 
+if ! [[ "$jobs" =~ ^[0-9]+$ ]] || [[ "$jobs" -lt 1 ]]; then
+  echo "-j needs a positive integer, got: $jobs" >&2
+  exit 2
+fi
+
 dir="${1:-}"
 if [[ -z "$dir" ]]; then
   echo "usage: $0 [-u URL] [-b] [-r] DIR" >&2
@@ -77,14 +86,33 @@ if [[ "$batch" -eq 1 ]]; then
   echo "-> HTTP $code"
   [[ "$code" == "202" ]]
 else
-  # One request per file.
+  # One request per file, up to $jobs in flight at once. Each upload runs in the
-  ok=0
+  # background so a slow (big) file doesn't stall the ones behind it. Every job
-  fail=0
+  # writes its HTTP status to a per-file line in $results; we tally after all
-  for f in "${files[@]}"; do
+  # jobs drain.
+  results="$(mktemp)"
+  trap 'rm -f "$results"' EXIT
+
+  # Upload one file: print progress and append "code<TAB>path" to $results.
+  upload_one() {
+    local f="$1" code
     code=$(curl -sS -o /dev/stderr -w '%{http_code}' -F "file=@${f}" "$endpoint")
     echo "  -> HTTP $code  ${f}"
-    if [[ "$code" == "202" ]]; then ok=$((ok + 1)); else fail=$((fail + 1)); fi
+    printf '%s\t%s\n' "$code" "$f" >> "$results"
+  }
+
+  for f in "${files[@]}"; do
+    # Throttle: wait for a slot to free up before launching the next upload.
+    # (|| true: a failed upload's exit status must not trip `set -e` here.)
+    while (( $(jobs -r -p | wc -l) >= jobs )); do
+      wait -n || true
+    done
+    upload_one "$f" &
   done
+  wait || true
+
+  ok=$(grep -c $'^202\t' "$results" || true)
+  fail=$(( ${#files[@]} - ok ))
   echo "done: $ok accepted, $fail failed"
   [[ "$fail" -eq 0 ]]
 fi