Added RabbitMQ

test/runtests.jl

@@ -22,6 +22,9 @@ using FileServer: Job, Config, ChannelQueue, enqueue!, dequeue!, close!, length,
                   Catalog, load_catalog, save_catalog!, catalog_sweep!, compact!,
                   write_nominations!, run_cluster_sweep, binary_files, record_example!,
                   signature_hex, ensure_dirs,
+                  ack!, nack!, with_delivery_tag, parse_backend, parse_bool, config_from_env,
+                  parse_amqp_url, job_json, job_from_message, AMQPTarget,
+                  RabbitQueue, connect_backend, open_queues, close_backend!,
                   MultipartReader, MultipartError, MultipartPart, next_part!,
                   write_part_body!, skip_part_body!, multipart_boundary,
                   parse_part_headers, spool_stream, UPLOAD_CHUNK_BYTES
@@ -1132,4 +1135,180 @@ end
         end
     end
 
+    @testset "queue backends" begin
+        @testset "backend selection is explicit or an error" begin
+            @test parse_backend("channel") === :channel
+            @test parse_backend("  RabbitMQ ") === :rabbitmq
+            # A typo must not quietly leave you on the in-process queue: that
+            # looks exactly like durability working, until a crash proves it isn't.
+            @test_throws ArgumentError parse_backend("rabbit")
+            @test_throws ArgumentError parse_backend("")
+
+            @test parse_bool("true") && parse_bool("1") && parse_bool("ON")
+            @test !parse_bool("false") && !parse_bool("0") && !parse_bool("off")
+            @test_throws ArgumentError parse_bool("maybe")
+
+            @test config_from_env().queue_backend === :channel   # default is unchanged
+            cfg = config_from_env(; queue_backend = :rabbitmq, amqp_prefetch = 4,
+                                    amqp_confirms = false, recover_spool = true)
+            @test cfg.queue_backend === :rabbitmq
+            @test cfg.amqp_prefetch == 4 && !cfg.amqp_confirms && cfg.recover_spool
+        end
+
+        @testset "amqp url parsing" begin
+            t = parse_amqp_url("amqp://user:pw@broker.internal:5673/prod")
+            @test t == AMQPTarget("broker.internal", 5673, "prod", "user", "pw")
+
+            # Bare form: default port, default vhost, and the guest credentials
+            # that only work over loopback anyway.
+            t2 = parse_amqp_url("amqp://localhost/")
+            @test t2.port == 5672 && t2.virtualhost == "/" && t2.login == "guest"
+            @test parse_amqp_url("amqp://localhost").virtualhost == "/"
+            # The conventional default vhost is often written percent-escaped.
+            @test parse_amqp_url("amqp://h:1/%2F").virtualhost == "/"
+            # Credentials may contain characters that need escaping in a URL.
+            @test parse_amqp_url("amqp://u%40b:p%2Fw@h/").password == "p/w"
+
+            @test_throws ArgumentError parse_amqp_url("http://localhost:5672/")
+            @test_throws ArgumentError parse_amqp_url("amqps://localhost:5671/")
+        end
+
+        @testset "the delivery tag never goes on the wire" begin
+            job = with_delivery_tag(Job("id-9", "a b.txt", "/tmp/spool/id-9-a_b.txt",
+                                        4096, 1234.5), 77)
+            @test job.delivery_tag == 77
+
+            # A tag names a delivery on one channel of one connection, not the
+            # job, so publishing it would be meaningless at best and would
+            # survive a restart as a lie at worst.
+            wire = JSON3.read(job_json(job))
+            @test !haskey(wire, :delivery_tag)
+            @test wire.id == "id-9" && wire.size == 4096 && wire.received_at == 1234.5
+
+            # Everything else must survive the round trip untouched, including a
+            # name with a space in it.
+            @test wire.original_name == "a b.txt"
+            @test wire.path == "/tmp/spool/id-9-a_b.txt"
+        end
+
+        @testset "acks are a no-op on the in-process queue" begin
+            # ChannelQueue has no delivery to settle, so `worker_loop` calling
+            # these on every job must cost nothing and change nothing.
+            q = ChannelQueue(2)
+            job = Job("id-1", "a.bin", "/tmp/a.bin", 1, 0.0)
+            @test enqueue!(q, job)
+            @test ack!(q, job) === nothing
+            @test nack!(q, job) === nothing
+            @test length(q) == 1
+        end
+
+        @testset "a duplicate delivery is settled, not quarantined" begin
+            # At-least-once means the same file can be handed to two workers, and
+            # the loser finds it already committed. That is a success it arrived
+            # too late for, not a failure: quarantining would file a `failed/`
+            # entry against a file that worked, and counting it either way would
+            # double-count one file.
+            mktempdir() do root
+                cfg = tmp_config(root)
+                q = ChannelQueue(10)
+                stats = StageStats()
+
+                dup = joinpath(cfg.spool_dir, "id-1-gone.bin")
+                write(dup, "x" ^ 10)
+                @test enqueue!(q, Job("id-1", "gone.bin", dup, 10, 0.0))
+
+                real = joinpath(cfg.spool_dir, "id-2-here.bin")
+                write(real, "y" ^ 10)
+                @test enqueue!(q, Job("id-2", "here.bin", real, 10, 0.0))
+                close!(q)
+
+                worker_loop(1, cfg, q, (job, _, _) -> begin
+                    # The duplicate's file was committed by the winner before the
+                    # handler ran; the other job fails with its file still there.
+                    job.id == "id-1" && rm(job.path)
+                    error("handler blew up")
+                end, stats)
+
+                @test stats.completed[] == 0
+                @test stats.failed[] == 1              # only the genuine failure
+                @test stats.bytes[] == 10              # the duplicate records nothing
+                @test stats.in_flight[] == 0
+                @test !isfile(joinpath(cfg.failed_dir, "id-1-gone.bin"))
+                @test isfile(joinpath(cfg.failed_dir, "id-2-here.bin"))
+            end
+        end
+
+        # The rest needs a live broker. Point FS_TEST_AMQP_URL at one to run it:
+        #   docker compose -f docker-compose.yml -f docker-compose.rabbitmq.yml up -d rabbitmq
+        #   FS_TEST_AMQP_URL=amqp://fileserver:fileserver@localhost:5672/ julia --project=. -e 'using Pkg; Pkg.test()'
+        amqp_url = get(ENV, "FS_TEST_AMQP_URL", "")
+        if isempty(amqp_url)
+            @info "skipping RabbitMQ integration tests (set FS_TEST_AMQP_URL to run them)"
+        else
+            @testset "rabbitmq round trip, redelivery and depth" begin
+                # A prefix per run, so a leftover queue from a previous run can
+                # never make this pass (or fail) for the wrong reason.
+                prefix = "fstest-" * string(rand(UInt32); base = 16)
+                cfg = config_from_env(; queue_backend = :rabbitmq, amqp_url = amqp_url,
+                                        amqp_prefix = prefix, queue_capacity = 5,
+                                        worker_count = 2, known_worker_count = 2,
+                                        unknown_worker_count = 2, text_worker_count = 2)
+                backend = connect_backend(cfg)
+                queues = open_queues(backend, cfg)
+                q = queues.classify
+                @test q isa RabbitQueue
+                @test capacity(q) == 5
+
+                @testset "a job survives the wire intact" begin
+                    @test enqueue!(q, Job("id-1", "hello.txt", "/tmp/hello.txt", 123, 1.5))
+                    got = dequeue!(q)
+                    @test got.id == "id-1"
+                    @test got.original_name == "hello.txt"
+                    @test got.path == "/tmp/hello.txt"
+                    @test got.size == 123
+                    @test got.received_at == 1.5
+                    @test got.delivery_tag != 0   # a real delivery, ackable
+                    ack!(q, got)
+                end
+
+                @testset "an unacked job comes back after a crash" begin
+                    # The whole point of the backend: take a job, never ack it,
+                    # lose the connection the way a SIGKILL would, and find it
+                    # waiting on restart.
+                    @test enqueue!(q, Job("id-2", "again.txt", "/tmp/again.txt", 7, 2.5))
+                    taken = dequeue!(q)
+                    @test taken.id == "id-2"
+
+                    foreach(close!, values(queues))
+                    close_backend!(backend)
+                    sleep(1.0)
+
+                    backend2 = connect_backend(cfg)
+                    queues2 = open_queues(backend2, cfg)
+                    q2 = queues2.classify
+                    redelivered = dequeue!(q2)
+                    @test redelivered.id == "id-2"
+                    ack!(q2, redelivered)
+
+                    sleep(2 * 1.0 + 0.5)          # let the depth poller catch up
+                    @test length(q2) == 0
+
+                    # Advisory capacity: publishing without draining must start
+                    # refusing rather than let the queue grow without bound.
+                    refused = 0
+                    for i in 1:40
+                        enqueue!(q2, Job("f$i", "f$i", "/tmp/f$i", 1, 0.0)) && continue
+                        refused = i
+                        break
+                    end
+                    @test refused > 0
+                    @test length(q2) >= capacity(q2)
+
+                    foreach(close!, values(queues2))
+                    close_backend!(backend2)
+                end
+            end
+        end
+    end
+
 end