From e55129e3a4d5fc1ed8cffd34f31c261b0abf1265 Mon Sep 17 00:00:00 2001 From: Jeffrey Ward Date: Thu, 2 Jul 2026 14:13:57 -0400 Subject: [PATCH] Add Lux.jl file classifier (known/unknown) with offline trainer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each uploaded file is scored by a fixed-structure neural net that labels it known (resembling the training set) or unknown — novelty detection over the first 16 + last 16 bytes (scaled to [0,1]), Dense(32->64->16->2), argmax. - src/model.jl: shared architecture + byte->feature mapping (trainer + server) - src/classify.jl: load committed artifact, classify a file at inference - bin/train.jl: offline trainer, 1:1 blended negatives (random + grab-bag), seeded 80/20 split, writes model/classifier.jld2 - worker: classify (annotate-only) and log classification=known|unknown - config: FS_MODEL_PATH; server fails fast if the artifact is missing - deps: Lux, JLD2, Optimisers, Zygote --- Manifest.toml | 832 +++++++++++++++++++++++++++++++++++++++++- Project.toml | 8 + README.md | 56 ++- bin/train.jl | 137 +++++++ model/classifier.jld2 | Bin 0 -> 22065 bytes src/FileServer.jl | 10 + src/classify.jl | 49 +++ src/config.jl | 6 +- src/model.jl | 51 +++ src/worker.jl | 8 +- 10 files changed, 1145 insertions(+), 12 deletions(-) create mode 100644 bin/train.jl create mode 100644 model/classifier.jld2 create mode 100644 src/classify.jl create mode 100644 src/model.jl diff --git a/Manifest.toml b/Manifest.toml index f5cb26b..93ebca6 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,12 +2,105 @@ julia_version = "1.12.6" manifest_format = "2.0" -project_hash = "d36132e59396f5ef81ef788c38b4b0d09ff8c86b" +project_hash = "1d7ce552eaac13c97732dccac45c97d40527ec58" + +[[deps.ADTypes]] +git-tree-sha1 = "d9aaef7c63466eee4de23b4d9dad03629df54bea" +uuid = "47edcb42-4c32-4615-8424-f2b9edc5f35b" +version = "1.22.1" +weakdeps = ["ChainRulesCore", "ConstructionBase", "EnzymeCore"] + + [deps.ADTypes.extensions] + ADTypesChainRulesCoreExt = "ChainRulesCore" + ADTypesConstructionBaseExt = "ConstructionBase" + ADTypesEnzymeCoreExt = "EnzymeCore" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" +weakdeps = ["ChainRulesCore", "Test"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + +[[deps.Adapt]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "daa72978cd7a624246e894a4f4f067706d4e17e2" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "4.7.0" +weakdeps = ["SparseArrays", "StaticArrays"] + + [deps.Adapt.extensions] + AdaptSparseArraysExt = "SparseArrays" + AdaptStaticArraysExt = "StaticArrays" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.ArrayInterface]] +deps = ["Adapt", "LinearAlgebra"] +git-tree-sha1 = "75757da5d9f771ef5909fc84f81d2f9d24127315" +uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +version = "7.27.0" + + [deps.ArrayInterface.extensions] + ArrayInterfaceAMDGPUExt = "AMDGPU" + ArrayInterfaceBandedMatricesExt = "BandedMatrices" + ArrayInterfaceBlockBandedMatricesExt = "BlockBandedMatrices" + ArrayInterfaceCUDAExt = "CUDA" + ArrayInterfaceCUDSSExt = ["CUDSS", "CUDA"] + ArrayInterfaceChainRulesCoreExt = "ChainRulesCore" + ArrayInterfaceChainRulesExt = "ChainRules" + ArrayInterfaceFillArraysExt = "FillArrays" + ArrayInterfaceGPUArraysCoreExt = "GPUArraysCore" + ArrayInterfaceMetalExt = "Metal" + ArrayInterfaceReverseDiffExt = "ReverseDiff" + ArrayInterfaceSparseArraysExt = "SparseArrays" + ArrayInterfaceStaticArraysCoreExt = "StaticArraysCore" + ArrayInterfaceTrackerExt = "Tracker" + + [deps.ArrayInterface.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + BandedMatrices = "aae01518-5342-5314-be14-df237901396f" + BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + CUDSS = "45b445bb-4962-46a0-9369-b4df9d0f772e" + ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2" + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" + GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" [[deps.Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" version = "1.11.0" +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "b8651b2eb5796a386b0398a20b519a6a6150f75c" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "1.1.3" + + [deps.Atomix.extensions] + AtomixCUDAExt = "CUDA" + AtomixMetalExt = "Metal" + AtomixOpenCLExt = "OpenCL" + AtomixoneAPIExt = "oneAPI" + + [deps.Atomix.weakdeps] + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + OpenCL = "08131aa3-fb12-5dee-8b74-c09406e224a2" + oneAPI = "8f75cd03-7ff8-4ecb-9b8f-daf728133b1b" + [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" version = "1.11.0" @@ -17,57 +110,308 @@ git-tree-sha1 = "bbe1079eecf9c9fbb52765193ad2bae27ae09bc8" uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" version = "0.1.10" +[[deps.CPUSummary]] +deps = ["CpuId", "IfElse", "PrecompileTools", "Preferences", "Static"] +git-tree-sha1 = "f3a21d7fc84ba618a779d1ed2fcca2e682865bab" +uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" +version = "0.2.7" + +[[deps.ChainRules]] +deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "SparseInverseSubset", "Statistics", "StructArrays", "SuiteSparse"] +git-tree-sha1 = "3c190c570fb3108c09f838607386d10c71701789" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.73.0" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra"] +git-tree-sha1 = "12177ad6b3cad7fd50c8b3825ce24a99ad61c18f" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.26.1" +weakdeps = ["SparseArrays"] + + [deps.ChainRulesCore.extensions] + ChainRulesCoreSparseArraysExt = "SparseArrays" + +[[deps.ChunkCodecCore]] +git-tree-sha1 = "1a3ad7e16a321667698a19e77362b35a1e94c544" +uuid = "0b6fb165-00bc-4d37-ab8b-79f91016dbe1" +version = "1.0.1" + +[[deps.ChunkCodecLibZlib]] +deps = ["ChunkCodecCore", "Zlib_jll"] +git-tree-sha1 = "cee8104904c53d39eb94fd06cbe60cb5acde7177" +uuid = "4c0bbee4-addc-4d73-81a0-b6caacae83c8" +version = "1.0.0" + +[[deps.ChunkCodecLibZstd]] +deps = ["ChunkCodecCore", "Zstd_jll"] +git-tree-sha1 = "34d9873079e4cb3d0c62926a225136824677073f" +uuid = "55437552-ac27-4d47-9aa3-63184e8fd398" +version = "1.0.0" + [[deps.CodecZlib]] deps = ["TranscodingStreams", "Zlib_jll"] git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9" uuid = "944b1d66-785c-5afd-91f1-9de20f533193" version = "0.7.8" +[[deps.CommonSubexpressions]] +deps = ["MacroTools"] +git-tree-sha1 = "cda2cfaebb4be89c9084adaca7dd7333369715c5" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.1" + +[[deps.CommonWorldInvalidations]] +git-tree-sha1 = "f1697a56da59e8a2cefcbbfe71c13354a6f18c61" +uuid = "f70d9fcc-98c5-4d4a-abd7-e4cdeebd8ca8" +version = "1.1.0" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.18.1" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" version = "1.3.0+1" +[[deps.ConcreteStructs]] +git-tree-sha1 = "1988532cb3b4525bb718b99ba07e433bbf0e600b" +uuid = "2569d6c7-a4a2-43d3-a901-331e8e4be471" +version = "0.2.5" + [[deps.ConcurrentUtilities]] deps = ["Serialization", "Sockets"] git-tree-sha1 = "21d088c496ea22914fe80906eb5bce65755e5ec8" uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" version = "2.5.1" +[[deps.ConstructionBase]] +git-tree-sha1 = "b4b092499347b18a015186eae3042f72267106cb" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.6.0" + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseLinearAlgebraExt = "LinearAlgebra" + ConstructionBaseStaticArraysExt = "StaticArrays" + + [deps.ConstructionBase.weakdeps] + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.CpuId]] +deps = ["Markdown"] +git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406" +uuid = "adafc99b-e345-5852-983c-f28acb93d879" +version = "0.3.1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + [[deps.DataStructures]] deps = ["OrderedCollections"] git-tree-sha1 = "6fb53a69613a0b2b68a0d12671717d307ab8b24e" uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" version = "0.19.5" +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" version = "1.11.0" +[[deps.DiffResults]] +deps = ["StaticArraysCore"] +git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.1.0" + +[[deps.DiffRules]] +deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "79a2aca180a85c690c58a020d47b426954b590f8" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.16.0" + +[[deps.DispatchDoctor]] +deps = ["MacroTools", "Preferences"] +git-tree-sha1 = "42cd00edaac86f941815fe557c1d01e11913e07c" +uuid = "8d63f2c5-f18a-4cf2-ba9d-b3f60fc568c8" +version = "0.4.28" +weakdeps = ["ChainRulesCore", "EnzymeCore"] + + [deps.DispatchDoctor.extensions] + DispatchDoctorChainRulesCoreExt = "ChainRulesCore" + DispatchDoctorEnzymeCoreExt = "EnzymeCore" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" + +[[deps.DocStringExtensions]] +git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.5" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.7.0" + +[[deps.EnzymeCore]] +git-tree-sha1 = "971d7831cc85f43bc9f51d615a3f7f21270c2f1d" +uuid = "f151be2c-9106-41f4-ab19-57ee4f262869" +version = "0.8.21" +weakdeps = ["Adapt", "ChainRulesCore"] + + [deps.EnzymeCore.extensions] + AdaptExt = "Adapt" + EnzymeCoreChainRulesCoreExt = "ChainRulesCore" + [[deps.ExceptionUnwrapping]] deps = ["Test"] git-tree-sha1 = "d36f682e590a83d63d1c7dbd287573764682d12a" uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" version = "0.1.11" +[[deps.ExpressionExplorer]] +git-tree-sha1 = "5f1c005ed214356bbe41d442cc1ccd416e510b7e" +uuid = "21656369-7473-754a-2065-74616d696c43" +version = "1.1.4" + +[[deps.FastClosures]] +git-tree-sha1 = "acebe244d53ee1b461970f8910c235b259e772ef" +uuid = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" +version = "0.3.2" + +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "8e9c059d6857607253e837730dbf780b6b151acd" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.19.0" +weakdeps = ["HTTP"] + + [deps.FileIO.extensions] + HTTPExt = "HTTP" + [[deps.FileServer]] -deps = ["HTTP", "JSON3", "Oxygen"] +deps = ["HTTP", "JSON3", "Logging", "Oxygen", "UUIDs"] path = "." uuid = "b3f1c2d4-5e6a-4b7c-8d9e-0f1a2b3c4d5e" version = "0.1.0" +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.FillArrays]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "2f979084d1e13948a3352cf64a25df6bd3b4dca3" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.16.0" + + [deps.FillArrays.extensions] + FillArraysPDMatsExt = "PDMats" + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStaticArraysExt = "StaticArrays" + FillArraysStatisticsExt = "Statistics" + + [deps.FillArrays.weakdeps] + PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[deps.ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] +git-tree-sha1 = "2c5d0b0e12088cde2cf84afb2784415b1ea3dfee" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "1.4.1" +weakdeps = ["StaticArrays"] + + [deps.ForwardDiff.extensions] + ForwardDiffStaticArraysExt = "StaticArrays" + +[[deps.Functors]] +deps = ["Compat", "ConstructionBase", "LinearAlgebra", "Random"] +git-tree-sha1 = "60a0339f28a233601cb74468032b5c302d5067de" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.5.2" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" +version = "1.11.0" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "83cf05ab16a73219e5f6bd1bdfa9848fa24ac627" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.2.0" + [[deps.HTTP]] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "PrecompileTools", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] git-tree-sha1 = "51059d23c8bb67911a2e6fd5130229113735fc7e" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" version = "1.11.0" +[[deps.HashArrayMappedTries]] +git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae" +uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74" +version = "0.2.0" + +[[deps.IRTools]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "ae3dae80f39426a5e598374e929522285e6ba8d0" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.17" + +[[deps.IfElse]] +git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1" +uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" +version = "0.1.1" + [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" version = "1.11.0" +[[deps.IrrationalConstants]] +git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.6" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLD2]] +deps = ["ChunkCodecLibZlib", "ChunkCodecLibZstd", "FileIO", "MacroTools", "Mmap", "OrderedCollections", "PrecompileTools", "ScopedValues"] +git-tree-sha1 = "941f87a0ae1b14d1ac2fa57245425b23a9d7a516" +uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" +version = "0.6.4" + + [deps.JLD2.extensions] + UnPackExt = "UnPack" + + [deps.JLD2.weakdeps] + UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" + [[deps.JLLWrappers]] deps = ["Artifacts", "Preferences"] git-tree-sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e" @@ -103,6 +447,18 @@ deps = ["StyledStrings"] uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" version = "1.12.0" +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "MacroTools", "PrecompileTools", "Requires", "StaticArrays", "UUIDs"] +git-tree-sha1 = "a5b87110fa95d711355af44832497745aa93fb52" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.42" +weakdeps = ["EnzymeCore", "LinearAlgebra", "SparseArrays"] + + [deps.KernelAbstractions.extensions] + EnzymeExt = "EnzymeCore" + LinearAlgebraExt = "LinearAlgebra" + SparseArraysExt = "SparseArrays" + [[deps.LRUCache]] git-tree-sha1 = "5519b95a490ff5fe629c4a7aa3b3dfc9160498b3" uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" @@ -112,6 +468,31 @@ weakdeps = ["Serialization"] [deps.LRUCache.extensions] SerializationExt = ["Serialization"] +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.15.0+0" + +[[deps.LibGit2]] +deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.9.0+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "OpenSSL_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.3+1" + [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" version = "1.11.0" @@ -121,6 +502,22 @@ deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" version = "1.12.0" +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "bba2d9aa057d8f126415de240573e86a8f39d2a1" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "1.0.1" + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + + [deps.LogExpFunctions.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" + InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112" + [[deps.Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" version = "1.11.0" @@ -131,11 +528,177 @@ git-tree-sha1 = "f00544d95982ea270145636c181ceda21c4e2575" uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" version = "1.2.0" +[[deps.Lux]] +deps = ["ADTypes", "Adapt", "ArrayInterface", "ChainRulesCore", "ConcreteStructs", "DiffResults", "DispatchDoctor", "EnzymeCore", "FastClosures", "ForwardDiff", "Functors", "GPUArraysCore", "LinearAlgebra", "LuxCore", "LuxLib", "MLDataDevices", "MacroTools", "Markdown", "NNlib", "Optimisers", "PrecompileTools", "Preferences", "Random", "ReactantCore", "Reexport", "SciMLPublic", "Setfield", "Static", "StaticArraysCore", "Statistics", "UUIDs", "WeightInitializers"] +git-tree-sha1 = "b7654d9b1144792d7fa165add2e07434329e3193" +uuid = "b2108857-7c20-44ae-9111-449ecde12c47" +version = "1.31.4" + + [deps.Lux.extensions] + ComponentArraysExt = "ComponentArrays" + EnzymeExt = "Enzyme" + FluxExt = "Flux" + GPUArraysExt = "GPUArrays" + LossFunctionsExt = "LossFunctions" + MLUtilsExt = "MLUtils" + MPIExt = "MPI" + MPINCCLExt = ["CUDA", "MPI", "NCCL"] + MooncakeExt = "Mooncake" + ReactantExt = ["Enzyme", "Reactant"] + ReverseDiffExt = ["FunctionWrappers", "ReverseDiff"] + SimpleChainsExt = "SimpleChains" + TrackerExt = "Tracker" + ZygoteExt = "Zygote" + + [deps.Lux.weakdeps] + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" + FunctionWrappers = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e" + GPUArrays = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" + LossFunctions = "30fc2ffe-d236-52d8-8643-a9d8f7c094a7" + MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" + MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" + Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" + NCCL = "3fe64909-d7a1-4096-9b7d-7a0f12cf0f6b" + Reactant = "3c362404-f566-11ee-1572-e11a4b42c853" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + SimpleChains = "de6bee2f-e2f4-4ec7-b6ed-219cc6f6e9e5" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[[deps.LuxCore]] +deps = ["DispatchDoctor", "Random", "SciMLPublic"] +git-tree-sha1 = "9455b1e829d8dacad236143869be70b7fdb826b8" +uuid = "bb33d45b-7691-41d6-9220-0943567d0623" +version = "1.5.3" + + [deps.LuxCore.extensions] + ArrayInterfaceReverseDiffExt = ["ArrayInterface", "ReverseDiff"] + ArrayInterfaceTrackerExt = ["ArrayInterface", "Tracker"] + ChainRulesCoreExt = "ChainRulesCore" + EnzymeCoreExt = "EnzymeCore" + FluxExt = "Flux" + FunctorsExt = "Functors" + MLDataDevicesExt = ["Adapt", "MLDataDevices"] + ReactantExt = "Reactant" + SetfieldExt = "Setfield" + + [deps.LuxCore.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" + Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196" + MLDataDevices = "7e8f7934-dd98-4c1a-8fe8-92b47a384d40" + Reactant = "3c362404-f566-11ee-1572-e11a4b42c853" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.LuxLib]] +deps = ["ArrayInterface", "CPUSummary", "ChainRulesCore", "DispatchDoctor", "EnzymeCore", "FastClosures", "Functors", "KernelAbstractions", "LinearAlgebra", "LuxCore", "MLDataDevices", "Markdown", "NNlib", "Preferences", "Random", "Reexport", "SciMLPublic", "Static", "StaticArraysCore", "Statistics", "UUIDs"] +git-tree-sha1 = "6a6453d556f7bc3870d797657636b1ad5f45fd27" +uuid = "82251201-b29d-42c6-8e01-566dec8acb11" +version = "1.15.9" + + [deps.LuxLib.extensions] + AppleAccelerateExt = "AppleAccelerate" + BLISBLASExt = "BLISBLAS" + CUDAExt = "CUDA" + CUDAForwardDiffExt = ["CUDA", "ForwardDiff"] + EnzymeExt = "Enzyme" + ForwardDiffExt = "ForwardDiff" + LoopVectorizationExt = ["LoopVectorization", "Polyester"] + MKLExt = "MKL" + OctavianExt = ["Octavian", "LoopVectorization"] + OneHotArraysExt = ["OneHotArrays"] + ReactantExt = ["Reactant", "ReactantCore"] + ReverseDiffExt = "ReverseDiff" + SLEEFPiratesExt = "SLEEFPirates" + TrackerAMDGPUExt = ["AMDGPU", "Tracker"] + TrackerExt = "Tracker" + cuDNNExt = ["CUDA", "cuDNN"] + + [deps.LuxLib.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + AppleAccelerate = "13e28ba4-7ad8-5781-acae-3021b1ed3924" + BLISBLAS = "6f275bd8-fec0-4d39-945b-7e95a765fa1e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + LoopVectorization = "bdcacae8-1622-11e9-2a5c-532679323890" + MKL = "33e6dc65-8f57-5167-99aa-e5a354878fb2" + Octavian = "6fd5a793-0b7e-452c-907f-f8bfe9c57db4" + OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" + Polyester = "f517fe37-dbe3-4b94-8317-1923a5111588" + Reactant = "3c362404-f566-11ee-1572-e11a4b42c853" + ReactantCore = "a3311ec8-5e00-46d5-b541-4f83e724a433" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + SLEEFPirates = "476501e8-09a2-5ece-8869-fb82de89a1fa" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" + [[deps.MIMEs]] git-tree-sha1 = "c64d943587f7187e751162b3b84445bbbd79f691" uuid = "6c6e2e6c-3030-632d-7369-2d6c69616d65" version = "1.1.0" +[[deps.MLDataDevices]] +deps = ["Adapt", "Functors", "Preferences", "Random", "SciMLPublic"] +git-tree-sha1 = "29b00f22be6fd821a214760f0224329f21998a05" +uuid = "7e8f7934-dd98-4c1a-8fe8-92b47a384d40" +version = "1.17.10" + + [deps.MLDataDevices.extensions] + AMDGPUExt = "AMDGPU" + CUDAExt = "CUDA" + ChainRulesCoreExt = "ChainRulesCore" + ChainRulesExt = "ChainRules" + ComponentArraysExt = "ComponentArrays" + FillArraysExt = "FillArrays" + GPUArraysSparseArraysExt = ["GPUArrays", "SparseArrays"] + MLUtilsExt = "MLUtils" + MetalExt = ["GPUArrays", "Metal"] + OneHotArraysExt = "OneHotArrays" + OpenCLExt = ["GPUArrays", "OpenCL"] + ReactantExt = "Reactant" + RecursiveArrayToolsExt = "RecursiveArrayTools" + ReverseDiffExt = "ReverseDiff" + SparseArraysExt = "SparseArrays" + TrackerExt = "Tracker" + ZygoteExt = "Zygote" + cuDNNExt = ["CUDA", "cuDNN"] + oneAPIExt = ["GPUArrays", "oneAPI"] + + [deps.MLDataDevices.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2" + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" + FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" + GPUArrays = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" + MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f" + OpenCL = "08131aa3-fb12-5dee-8b74-c09406e224a2" + Reactant = "3c362404-f566-11ee-1572-e11a4b42c853" + RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" + oneAPI = "8f75cd03-7ff8-4ecb-9b8f-daf728133b1b" + +[[deps.MacroTools]] +git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.16" + [[deps.Markdown]] deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -161,6 +724,40 @@ version = "1.11.0" uuid = "14a3606d-f60d-562e-9121-12d972cd8159" version = "2025.11.4" +[[deps.NNlib]] +deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Random", "ScopedValues", "Statistics"] +git-tree-sha1 = "2acc74264095c06beb62c6f69691a3277b5ac3d0" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.9.37" + + [deps.NNlib.extensions] + NNlibAMDGPUExt = "AMDGPU" + NNlibCUDACUDNNExt = ["CUDA", "cuDNN"] + NNlibCUDAExt = "CUDA" + NNlibEnzymeCoreExt = "EnzymeCore" + NNlibFFTWExt = "FFTW" + NNlibForwardDiffExt = "ForwardDiff" + NNlibMetalExt = "Metal" + NNlibMooncakeCUDAExt = ["Mooncake", "CUDA"] + NNlibSpecialFunctionsExt = "SpecialFunctions" + + [deps.NNlib.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6" + SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" + cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "dbd2e8cd2c1c27f0b584f6661b4309609c5a685e" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.1.4" + [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" version = "1.3.0" @@ -170,6 +767,11 @@ deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" version = "0.3.29+0" +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.7+0" + [[deps.OpenSSL]] deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "NetworkOptions", "OpenSSL_jll", "Sockets"] git-tree-sha1 = "1d1aaa7d449b58415f97d2839c318b70ffb525a0" @@ -181,6 +783,28 @@ deps = ["Artifacts", "Libdl"] uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" version = "3.5.4+0" +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.6+0" + +[[deps.Optimisers]] +deps = ["ChainRulesCore", "ConstructionBase", "Functors", "LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "36b5d2b9dd06290cd65fcf5bdbc3a551ed133af5" +uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" +version = "0.4.7" + + [deps.Optimisers.extensions] + OptimisersAdaptExt = ["Adapt"] + OptimisersEnzymeCoreExt = "EnzymeCore" + OptimisersReactantExt = "Reactant" + + [deps.Optimisers.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + Reactant = "3c362404-f566-11ee-1572-e11a4b42c853" + [[deps.OrderedCollections]] git-tree-sha1 = "94ba93778373a53bfd5a0caaf7d809c445292ff4" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" @@ -216,6 +840,17 @@ git-tree-sha1 = "32a4e09c5f29402573d673901778a0e03b0807b9" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "2.8.6" +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.12.1" + + [deps.Pkg.extensions] + REPLExt = "REPL" + + [deps.Pkg.weakdeps] + REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + [[deps.PrecompileTools]] deps = ["Preferences"] git-tree-sha1 = "edbeefc7a4889f528644251bdb5fc9ab5348bc2c" @@ -238,6 +873,18 @@ deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" version = "1.11.0" +[[deps.ReactantCore]] +deps = ["ExpressionExplorer", "MacroTools"] +git-tree-sha1 = "f8b4bdc56e1ab25053e525095e67a75dd48c95e4" +uuid = "a3311ec8-5e00-46d5-b541-4f83e724a433" +version = "0.1.20" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + [[deps.Reexport]] git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" uuid = "189a3867-3050-52da-a836-e630ba90ab69" @@ -249,10 +896,27 @@ git-tree-sha1 = "ffdaf70d81cf6ff22c2b6e733c900c3321cab864" uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" version = "1.0.1" +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.1" + [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" version = "0.7.0" +[[deps.SciMLPublic]] +git-tree-sha1 = "2b1b64add566435a768abdb3b053cac17d19ff3c" +uuid = "431bcebd-1456-4ced-9d72-93c2757fff0b" +version = "1.2.1" + +[[deps.ScopedValues]] +deps = ["HashArrayMappedTries", "Logging"] +git-tree-sha1 = "67a144433c4ce877ee6d1ada69a124d6b1ecf7be" +uuid = "7e506255-f358-4e82-b7e4-beb19740aa63" +version = "1.6.2" + [[deps.Scratch]] deps = ["Dates"] git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109" @@ -263,6 +927,12 @@ version = "1.3.0" uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" version = "1.11.0" +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "c5391c6ace3bc430ca630251d02ea9687169ca68" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.2" + [[deps.SimpleBufferStream]] git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" @@ -272,17 +942,72 @@ version = "1.2.0" uuid = "6462fe0b-24de-5631-8697-dd941f90decc" version = "1.11.0" +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.12.0" + +[[deps.SparseInverseSubset]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "52962839426b75b3021296f7df242e40ecfc0852" +uuid = "dc90abb0-5640-4711-901d-7e5b23a2fada" +version = "0.1.2" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "6547cbdd8ce32efba0d21c5a40fa96d1a3548f9f" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.8.0" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.Static]] +deps = ["CommonWorldInvalidations", "IfElse", "PrecompileTools", "SciMLPublic"] +git-tree-sha1 = "b151f033556272891e184d7d36c62518b56bbaac" +uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" +version = "1.4.2" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "246a8bb2e6667f832eea063c3a56aef96429a3db" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.18" +weakdeps = ["ChainRulesCore", "Statistics"] + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "6ab403037779dae8c514bad259f32a447262455a" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.4" + [[deps.Statistics]] deps = ["LinearAlgebra"] git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" version = "1.11.1" +weakdeps = ["SparseArrays"] [deps.Statistics.extensions] SparseArraysExt = ["SparseArrays"] - [deps.Statistics.weakdeps] - SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +[[deps.StructArrays]] +deps = ["ConstructionBase", "DataAPI", "Tables"] +git-tree-sha1 = "ad8002667372439f2e3611cfd14097e03fa4bccd" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.7.3" +weakdeps = ["Adapt", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "SparseArrays", "StaticArrays"] + + [deps.StructArrays.extensions] + StructArraysAdaptExt = "Adapt" + StructArraysGPUArraysCoreExt = ["GPUArraysCore", "KernelAbstractions"] + StructArraysLinearAlgebraExt = "LinearAlgebra" + StructArraysSparseArraysExt = "SparseArrays" + StructArraysStaticArraysExt = "StaticArrays" [[deps.StructTypes]] deps = ["Dates", "UUIDs"] @@ -310,11 +1035,37 @@ version = "2.8.2" uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" version = "1.11.0" +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.8.3+2" + [[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" version = "1.0.3" +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "0f38a06c83f0007bbab3cf911262841c9a0f07e0" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.13.0" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" @@ -339,12 +1090,85 @@ version = "1.11.0" uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" version = "1.11.0" +[[deps.UnsafeAtomics]] +git-tree-sha1 = "0f30765c32d66d58e41f4cb5624d4fc8a82ec13b" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.3.1" + + [deps.UnsafeAtomics.extensions] + UnsafeAtomicsLLVM = ["LLVM"] + + [deps.UnsafeAtomics.weakdeps] + LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" + +[[deps.WeightInitializers]] +deps = ["ConcreteStructs", "GPUArraysCore", "LinearAlgebra", "Random", "SpecialFunctions", "Statistics"] +git-tree-sha1 = "d60c958a009b5a25cf8ea4eb3f9a2cf7352a9c6f" +uuid = "d49dbf32-c5c2-4618-8acc-27bb2598ef2d" +version = "1.3.4" + + [deps.WeightInitializers.extensions] + AMDGPUExt = "AMDGPU" + CUDAExt = "CUDA" + ChainRulesCoreExt = "ChainRulesCore" + GPUArraysExt = "GPUArrays" + ReactantExt = "Reactant" + + [deps.WeightInitializers.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + GPUArrays = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" + Reactant = "3c362404-f566-11ee-1572-e11a4b42c853" + [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" version = "1.3.1+2" +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "446b23e73536f84e8037f5dce465e92275f6a308" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.7+1" + +[[deps.Zygote]] +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "GPUArraysCore", "IRTools", "InteractiveUtils", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NaNMath", "PrecompileTools", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "615fae83fbb607abc43cf2f84f51bcf3be1a706b" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.7.11" + + [deps.Zygote.extensions] + ZygoteAtomExt = "Atom" + ZygoteCUDAExt = "CUDA" + ZygoteColorsExt = "Colors" + ZygoteDistancesExt = "Distances" + ZygoteTrackerExt = "Tracker" + + [deps.Zygote.weakdeps] + Atom = "c52e3926-4ff0-5f6e-af25-54175e0327b1" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" + Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.ZygoteRules]] +deps = ["ChainRulesCore", "MacroTools"] +git-tree-sha1 = "434b3de333c75fc446aa0d19fc394edafd07ab08" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.7" + [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" version = "5.15.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.64.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.7.0+0" diff --git a/Project.toml b/Project.toml index eea2254..672353b 100644 --- a/Project.toml +++ b/Project.toml @@ -5,14 +5,22 @@ authors = ["wardjm@gmail.com"] [deps] HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +Lux = "b2108857-7c20-44ae-9111-449ecde12c47" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] HTTP = "1.11.0" +JLD2 = "0.6.4" JSON3 = "1.14.3" Logging = "1.11.0" +Lux = "1.31.4" +Optimisers = "0.4.7" Oxygen = "1.10.2" UUIDs = "1.11.0" +Zygote = "0.7.11" diff --git a/README.md b/README.md index a3ade6b..f5538fe 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,9 @@ pool of worker threads for processing. The HTTP endpoint does no real work: it spools each uploaded file to disk, pushes a lightweight reference onto a work queue, and responds immediately — staying free to accept the next upload. -Right now the "processing" is just logging the received filename, to prove the -flow. That's the seam where real heavy-lifting goes later. +The per-file "processing" runs each file through a small neural-network +classifier that labels it **known** (a file type resembling the training set) or +**unknown**, and logs the result. See "File classifier" below. ## Architecture @@ -82,6 +83,49 @@ Both SIGINT and SIGTERM trigger the same idempotent graceful drain `STOPSIGNAL SIGINT`). Give the stop timeout enough headroom to drain in-flight work (systemd: `TimeoutStopSec`). +## File classifier + +Each file is scored by a fixed-structure neural network (Lux.jl) that answers a +single binary question: is this file **known** (like the types in the training +set) or **unknown**? It's novelty detection, not exact file-typing — it won't +tell you "PDF", just "this looks like something I was trained on, or not". + +- **Features:** the first 16 bytes + last 16 bytes of the file, each scaled + 0–255 → `[0,1]`, giving a 32-dim input. Files under 32 bytes can't form that + window and are classified `unknown` without touching the model. +- **Architecture:** `Dense(32→64,relu) → Dense(64→16,relu) → Dense(16→2)`, + raw logits; decision is `argmax` (class 1 = known, class 2 = unknown). +- **Artifact:** trained weights live in `model/classifier.jld2` (committed), so + the server just loads them at startup. Missing/unreadable ⇒ the server fails + fast rather than run without classification. +- **Effect today:** *annotate-only*. The class is logged + (`classification=known|unknown`) but every file still moves to `done/`; the + classifier can't misroute real files while it's unproven. + +The architecture and byte→feature mapping are defined once in `src/model.jl` and +shared by the trainer and the server, so they can't drift apart. + +### Training + +Training is a separate, offline script — it never runs in the request path: + +```bash +julia --project=. bin/train.jl [negatives_dir] +``` + +- **positives_dir** — every file in it (≥32 bytes) is a "known" example. +- **negatives_dir** *(optional)* — a grab-bag of *other* real file types used as + "unknown" examples. Negatives are generated ~1:1 with positives, split 50/50 + between uniform-random byte vectors and grab-bag files. With no grab-bag dir, + negatives are all random (weaker: the net may just learn "high entropy = + unknown" rather than your actual types, so a grab-bag of real off-distribution + files is recommended). + +The script uses an 80/20 seeded split, reports validation accuracy, and writes +`model/classifier.jld2` (path overridable via `FS_MODEL_PATH`). A fixed seed +(`FS_TRAIN_SEED`, default 42) drives negative generation, the split, and weight +init, so the artifact is exactly regenerable from the same inputs. + ## Configuration (environment variables) | Variable | Default | Meaning | @@ -93,6 +137,7 @@ Both SIGINT and SIGTERM trigger the same idempotent graceful drain | `FS_SPOOL_DIR` | `data/spool` | Incoming files (pending) | | `FS_DONE_DIR` | `data/done` | Files after successful processing | | `FS_FAILED_DIR` | `data/failed` | Files whose processing threw | +| `FS_MODEL_PATH` | `model/classifier.jld2` | Classifier artifact loaded at startup | > To get real parallelism, start Julia with enough threads (`-t N`) to match > `FS_WORKERS`. If `FS_WORKERS` exceeds available threads you'll get a warning @@ -126,8 +171,13 @@ src/ job.jl Job (the queue reference) queue.jl JobQueue seam + in-process ChannelQueue spool.jl filename sanitizing, spool/move, startup recovery - worker.jl worker loop + per-job processing (placeholder) + model.jl NN architecture + byte→feature mapping (shared with trainer) + classify.jl load artifact + classify a file at inference time + worker.jl worker loop + per-job processing (classify + move) server.jl HTTP routes/handlers bin/ server.jl entry point + train.jl offline training script → model/classifier.jld2 +model/ + classifier.jld2 committed trained weights (loaded at startup) ``` diff --git a/bin/train.jl b/bin/train.jl new file mode 100644 index 0000000..9c26151 --- /dev/null +++ b/bin/train.jl @@ -0,0 +1,137 @@ +#!/usr/bin/env julia +# +# Train the file-type novelty classifier and write the artifact the server loads. +# +# julia --project=. bin/train.jl [negatives_dir] +# +# positives_dir every file in it (>=32 bytes) is a "known" example (class 1). +# negatives_dir optional grab-bag of *other* real file types used as "unknown" +# (class 2). If omitted/empty, unknown examples are all random. +# +# Negatives total ~1:1 with positives, split 50/50 between uniform-random byte +# vectors and grab-bag files (falling back to all-random when no grab-bag). +# One fixed seed (FS_TRAIN_SEED, default 42) drives negatives, the train/val +# split, and weight init, so the committed artifact is exactly regenerable. +# +# The model is trained on an 80% split; validation accuracy on the held-out 20% +# is reported so you can see whether it actually learned. The 80%-trained model +# is what gets saved (no retrain-on-all for this test). + +using Lux +using JLD2 +using Optimisers +using Zygote +using Random +using Statistics + +include(joinpath(@__DIR__, "..", "src", "model.jl")) # build_model(), read_features(), FEATURE_DIM, MIN_FILE_BYTES + +const EPOCHS = 200 +const LEARNING_RATE = 1f-3 +const VAL_FRACTION = 0.20 + +# --- data assembly ----------------------------------------------------------- + +"List regular files directly under `dir` that are large enough to featurize." +function eligible_files(dir::AbstractString) + (isempty(dir) || !isdir(dir)) && return String[] + paths = String[] + for name in readdir(dir; join=true) + isfile(name) && filesize(name) >= MIN_FILE_BYTES && push!(paths, name) + end + return paths +end + +"32×N Float32 feature matrix from a list of file paths (all assumed eligible)." +function feature_matrix(paths::Vector{String}) + isempty(paths) && return Matrix{Float32}(undef, FEATURE_DIM, 0) + return reduce(hcat, (read_features(p) for p in paths)) +end + +# logsoftmax over the class dimension (rows), numerically stabilized. Local so +# the trainer needs no extra dependency for the loss. +function logsoftmax(x::AbstractMatrix) + m = maximum(x; dims=1) + shifted = x .- m + return shifted .- log.(sum(exp.(shifted); dims=1)) +end + +logitcrossentropy(logits, onehot) = mean(-sum(onehot .* logsoftmax(logits); dims=1)) + +accuracy(logits, onehot) = mean(vec(map(i -> i[1], argmax(logits; dims=1))) .== + vec(map(i -> i[1], argmax(onehot; dims=1)))) + +function main() + if isempty(ARGS) + println(stderr, "usage: julia --project=. bin/train.jl [negatives_dir]") + exit(2) + end + positives_dir = ARGS[1] + negatives_dir = length(ARGS) >= 2 ? ARGS[2] : "" + seed = parse(Int, get(ENV, "FS_TRAIN_SEED", "42")) + rng = Random.MersenneTwister(seed) + + pos_paths = eligible_files(positives_dir) + P = length(pos_paths) + P == 0 && error("no eligible (>= $MIN_FILE_BYTES byte) files found in positives dir: $positives_dir") + + grabbag = eligible_files(negatives_dir) + + # Negatives ~1:1 with positives, 50/50 random vs grab-bag (all-random if the + # grab-bag is empty). Grab-bag files are sampled with replacement so we can + # always hit the target count from a small pile. + n_neg = P + n_grab = isempty(grabbag) ? 0 : n_neg ÷ 2 + n_rand = n_neg - n_grab + + Xpos = feature_matrix(pos_paths) + Xrand = rand(rng, Float32, FEATURE_DIM, n_rand) # uniform [0,1) ≈ random bytes / 255 + grab_paths = n_grab == 0 ? String[] : rand(rng, grabbag, n_grab) + Xgrab = feature_matrix(grab_paths) + + X = hcat(Xpos, Xrand, Xgrab) + N = size(X, 2) + # One-hot targets: known = [1,0], unknown = [0,1]. + Y = zeros(Float32, 2, N) + Y[1, 1:P] .= 1f0 # positives -> known + Y[2, P+1:end] .= 1f0 # all negatives -> unknown + + @info "assembled dataset" positives=P negatives=n_neg random_neg=n_rand grabbag_neg=n_grab total=N grabbag_pool=length(grabbag) + + # Seeded shuffle + 80/20 split. + perm = randperm(rng, N) + X, Y = X[:, perm], Y[:, perm] + n_val = round(Int, VAL_FRACTION * N) + n_train = N - n_val + Xtr, Ytr = X[:, 1:n_train], Y[:, 1:n_train] + Xval, Yval = X[:, n_train+1:end], Y[:, n_train+1:end] + + # --- train (full-batch, fixed epochs) ------------------------------------ + model = build_model() + ps, st = Lux.setup(rng, model) + opt_state = Optimisers.setup(Optimisers.Adam(LEARNING_RATE), ps) + + for epoch in 1:EPOCHS + (loss, st), back = Zygote.pullback(ps) do p + logits, st_new = model(Xtr, p, st) + logitcrossentropy(logits, Ytr), st_new + end + grads = back((one(loss), nothing))[1] + opt_state, ps = Optimisers.update(opt_state, ps, grads) + if epoch == 1 || epoch % 20 == 0 || epoch == EPOCHS + tr_logits, _ = model(Xtr, ps, st) + @info "epoch" epoch loss=loss train_acc=accuracy(tr_logits, Ytr) + end + end + + # --- report + save ------------------------------------------------------- + val_logits, _ = model(Xval, ps, st) + @info "validation" n_val=n_val val_acc=(n_val == 0 ? NaN : accuracy(val_logits, Yval)) + + out = get(ENV, "FS_MODEL_PATH", "model/classifier.jld2") + mkpath(dirname(out)) + jldsave(out; ps=ps, st=st) + @info "saved model artifact" path=out +end + +main() diff --git a/model/classifier.jld2 b/model/classifier.jld2 new file mode 100644 index 0000000000000000000000000000000000000000..6768d4382cd7fac918a677e7546ab75675747bbe GIT binary patch literal 22065 zcmeIa30RF?_c(l-=cG|dgM>m#igfn6LzyGWObJOuBorZyC@LfxP*P-wLaDRYtk{LfMSUT>H8`JVT8{jcx3zVCYW)oGu#*4}Haz1F_hx%ayFz9(5vFdMei zd4tPxohcjL-JEr-oxPlOCU|(RarPRjv&qGCgPX@%9b=;rM#dbS!JSP^j7&xtnd_L3 z9JbWWOJ|ym0q39c|5Xd@o@6~iPFh1+SW=S15$+tt&#q!JrxQ-H@2vIdnuRkaS{z}$AsLP|N0hTlF!P+eeAc;eM1%zl1Vx~X9RE`+0QjYoY3KLu|K>4L(>lLT96w{) zuNDFs{36uVUcf<6#(Q|W7zyUKgFuRadndnMzo=dG>3nUMJ8WF%&ex-hF7rFx-e=8H z5BFaf(Js1p7kzqXH;z~rzDo_x1I5k;vwC;d;$8Ia(U^4gm*{~X(S?`nA!oCTo9jw1 z4o9jBFWS{^shjf#!B^=nymS}-*SNcM6QE@NL?7$v>Fm?lLAHx7`+NTh?jFuwroU8@ z>!Np$&r!LvVQyC~|Np?h*MI20UZsl$CTX_UM0eY)`7{_0t)bhYd0Kh+-iZvUzEz?*jQ@6&~s=z`syeOx@9OgQR)#E;-; z{1I=;>Dz_xIUda}eD`>&yByZD?$_0>=XkVw;Je48-2>k}9-S_H_q<5#60D-$IY8Ym zde8Cp|08|`NAHh#Q_g@c{D1PV{vZ39a0dPnKY}ypk9bqg;4XYmyBc)i|C4;f|Ily9 zf4~pz!gt#(d}gP;?sV0|y6D}0JZINQ?^zr5;8QiDv)z=g_MH%i^SnDSbR+($|FOV7 z7Wl^k|5)H33;bh&e=P8i1^%(X{|_wCnWyj&8Q7Vp5ZqpOFP*zH5WkkplHJhX%0~$9 z`n&THg8YQw(+vw2_&%klR(DEXETYV@`cqEiZ^;p49lH67@W1yg0o!Ko|0PFANl3)} zU#S>=HsMb)NAS}xe@M^#EmZ^wj&4;1$%fyDELa@LItXyG9A*Azg`11}awkDPXu}`% z5awrP1pNg?u;A?GE0FO!ELbGUj_%|u`9FLG`JoN}lRx^MSISh3;fP6 zk^=;+bU44;OCUyIv+i+92^w~<^aS6#%2LjL-%fp9MYR4kVfo(do}D8&|J_6ttn#|0 z3cd>l*bNI5==0gttyAJ32W8A}A}HO1`kkF%ZvM{hSBC8mgX+yNIsb(SLBgq9!|vph zK!yJ)e-N=|#hI4>xBUE9CXweW`MXJeB?tbXuaJSQ^M9ppcUI<4lV-}AIj?^)6WSvr z62+f2{6Ha4vOAN+5#(nDO8j-)>uk=C5dTecQ;yExYyP)F%vZ2mZY&|F7DSO+&-g4w(D1ZN!Lu{Un|1=r$jnkeXP1_{(YU0 ziH`QK-*pa0hodIRQQN_%N%A9?sE~J;Dttm``NIs`{`sgVXTSLgwBL1zH_9{s&dP@3 zt-eDMF4$sjYY~>+xP{xpzmxHACE2@$w_t(uV;t~S4lC_yF|}VY-D}W?z2do=Dt0K6 zRd;8={(43BZp~9V&r_5+`=EmOU$L;ky1wZ%o}32SUN=o3M88P9%Nh>1yAT@S<6q&6;@=Z%w}sOYc30oy&HEWBF$oy*L{! zhP{RGpgKICAB9!2Lz(Cgl~)t)4ut%%dwKCS%`j-B3%q?~jR|KD;g@4SU_-y-DErX{ zlS4g0;e;M5t@DMvNxzSqL7FX*-HD2W)M3pqOISB_Ed4ac6rFyqrEcSO!0??lkTKzS z-rW-=-Q^gG$w_c;dldv4)zC*BlFT=akub`=7-P$BVP>!{vuDF(*o78k%6A!dLZcBQ zcKA5liv9s(uPL$C6JlZcwshnwsj`dor;`KH;;h?(_q5LY5^iXHX~`UJBjM6=%*vB) z^ljO0xZ*z+pV?lAW!39pZ|h@P8?Mi8u~lQsu06u7j1ESI+hD|lBI@(gkyeE3G5Hs+ zQD2d4jD5X^^vyZ~TX&aZu&gTM`r;{AJrQNa)-_V$spT+-4Iw9<<=`t3bsSSH%$kiT zqux{1VT@)827T(y-dZph^ipN$iNeeL`Ei$MH2UDu-LlLYFT}RvZwWUp&~j5<5;&R; z#@Q>Ez=IhQY?g$TWpn8eCiuyDZsIg`_C2V^v zN{BtF@PbUfsKyR_;EbN}r*L4M2vu*4Mib{s6jI89nkprDawHL6Iv*$Vt&edfIEh5g zs2RiCZj+j(PmmgU8Qu4hV0^r&mTGEI20@-@60@6}j6zkL`_l0f-+HhRi9!TMVZ&_Vt?GzZEt zrY*|&a`JpKAXT3E`i=`z3$pO+N-b=D7fg-K-EiBzP_%2~BI-oYcZb!PUcn;llL9jc zSw9SFRp-D*J#F?{_FITK8iwX0DHiw^z$(8c9RB(>?(+%2=G1f$G0njn!Y2#)g;VoT}5&V+?)# z=fe)k6S#VM5gB2U0pfla@JHeqY{T2=sNBp8zL5rvlVzDr&a+Wy#u&)XQf1Pk*V0P~ zXLv(asj*SVUxCce5p3dH8D{#X*BI&e5;e+ViTJY)+Q^fFPnpWBh-Vhn5e)%uWE!3b zP-fb1i?eH7D9P*R2-TMQ$SE>r6z65pvTz`FMte{WZ@?FgK42~-&WfW1E499kE@>*o zjfW+108fXVD<#H^nrF<$?GR$zhfg8)FO-K@e^Ewj`3)F(=pEJ6 zmS+sO-$7(#<#hChGq8RsCgo!gefQfP6A<01$(>C3P@b-Kdp*I`ar-k5%qb;a= zQJyI-*-tB9m@_WJL>T1{ZLmb44U*Eccut}6?0pLpoZ*~{7yIe5ZSOUCmyBAmj4Mg4 zeTNg{v<_149fI#ZZHFaMn(Vvv0o1{-ireS%Ll_q~om5>rgf+6e@b2B^FmqJ{R4MHx z532KN^q!wE-FzY_ypmwpk_H@To&W_-W!#2c@pQ`B z;Z9&I5dR|>VqJxH{nXisZ?mv0_aRDGe8r}?I8ZcgA{QP@GM7&GW`i; z;7*(l&&J21){;ETNsEFy5oLIP(1aBaR%D+nFoSVY>P+-fU1kD!lYtHjEW1`3kJzd) zE8J_yfPTH$jDDXW&qR*hX{Lmy17+FNOKotVVi+7%YlNZOBT)T#C73Q%#nX&2ns)@E z{-$FXnWx48aQ)Ej5V-&3>8;Y*z<);alp`RWYcp$s97nG*JVCnbcZQ^ zT(k+l&l=536u%0&F`@WLLXOq1PUXjaeKvS)0WO<32V2GaFkc;IadEH>y_0&1%-|Wa z=I$KU|KbS@TqnZ%&8@;Oa+<8}`tvxzuYf1N={n9>IG%n|9m1|&DZ!rf?ZYms8i033 z$S^5+djLZ$DK|3&wr$OWfXA^gV2=U2VOlb2Zt>&w9<`A$(!H1kZb3LQeg`g$1>+gZ$ZhiJb#Q~gi zVgxf#<1@%!7G?C$#gJT4c}C%69E4l%hDGNPENad`#q%xTu;B;>m6br*xBa{m0aqa* z)(P}S<-@Y+mdujl5-jJ*WJo;Y0TZ{3KwEWvqP)KtlKbt%=VL0!rwTbX#wiXauK;3G zJ_;4nrTMzt#Ve8;4A(6j2ft}RyPHd}fE!8gCF(Q9i`U^r^INbYg@xyrLh;(H>ENeg z4GEiN*elUBSdc1)M-D#28gDW7wRi|*C$`bm6Ku%Dni7-_I)okGgRwj?lpEa<1iY|b zOh)Ajta_hBUc2itX3es!g_ z^1R^?ET|V|PtM;@ELWVSGnEctjjuQ}gR=x+p(wL3-Umw$6@#|Ie%_wwtr>oZ$d3%7!y5 z-iWhvRF9Gl=`2tt^FjDgJ<+=U6iS;fzlQYCfJr@riC2ce>fOd zy;o)In~p+Q`&DeNP-0&Bj-&@eGa=YEz_NOR6w~th4zB1g&lnEzf(a7~cydDuan;&b zR8-5yi`g&9#dZ@|u>Ubm<8qi&ThCLQco8P`h!_*r!Eim?=D^*=CQ@&51@B6gk*A`& zAb&k!SG_)nk22(#s$*iT?OrK%RJ0T$^mGdx3^@j0&otqqU9J2WQ_5}C6lYx?^n$eg z((q)yJ;qN=Cdqd36ucbi@$VegzK;ZJf5;HU7Fpo45(0;;FQ9ssA}tN_qt9RGP=l!) z;7sA+Fi>Nrgs!0%mgqvZ(J4IsC;)=?CBu<}=P=(*9A4IJgwRYeP>H!t8-0F^nR71; z^Qwk0H>3<0O~0W`=K9N|W8oxH8m>yO2{(bx?7fyFZZG3-uMd2w-0C8bH3F4 z`Beyr9m+gEQiH2*wSd^2(GZiR&d%FdjAP|g$&V*<@NGsY*CPA87yV401bLnwdH<4{L>n(lrf3L0R+$8mnlr9b%gJVwVf9xMIv4 z6g~oj%7xe~Z*Snbr)dy#QG`j~QGu<|M=?i(zzCHV`hBy9RcS7(n}KSiqviAE0^SLp<&#%`9?_qgzTOKt5~`?`_UGtY%Nc=oyX>61kq5 z91&-(tq($-r4C9AyfSW4=VP3%} zP~WS_F7++ITnQa8@HfWh=`~QGdl|1cXfajRO1Qlt627{Q;+@$a4YqyNP`l4IGCi~o z#xpUPQX5e+{f5Kp+~MH%atPbA9%tWu0*#uA*r6}MYM5p~ zx#1mrG}R62%UW?os0h0-SeA(zJqIL)_rj8>;dt+(7kW-8!`hlVkofdI%2#Xw?Gz!V znTWBGYaZdpf=IaMsKV$RZl)sd_Q0kaUvsvZd*il_I_}u5Ct)RrgPiEMbV!Rd`=xa{ z3Mcx(xZpBqo%RCf%{IXYAD>|nX8?1>xP{&u>`$bp6+=hB736)BN0$mj$)Q=am;Wnj zn01D_Z`cj9wp=4$G|t0d%`h^3ZwijND#9GNsLXQIw8)LA$4Cd)fQ)~XiC-pG^W%^# zE^Dx+&mE+hq|&{ZkRXW_AzNU5+C6l7Q41xB7HA8a?CAbW!CNa8&mP^w(On-QqfkqYbEkQxgO>&*n=&l zz1ZbWt7)nEb1Yo49e3#)Q8T5{@VLGhQpYBdX%EENJ|5Dnb^jF*9=-rl%F9v8c`^!R zI3Taje%SX_j@jWU$qafa!kU>r1J<`G$5f#LOe$}o{TN4pu|H^fpd*eN)PSSq;&7(e zJ20!x&9SZTO{8wD0G*0j)Y)?d+vI+t_M8|vRUU*+(|fV!g%6{j`X@ZR^Ds&ew;Ln; z`6I4<+zPEShxz%ReEz+)2wQESM28zz!GWfiILJ2w&ZKKWOS>6lD88f1CLV`tR;tVi z-!fe5&0ovezInu>~JOVTYOw z^Xy4H@Qk@opnidfNhiZQtpg|`p$fWpR{(ExU!2+N9X=Sc8~3-(!=G-FENj04)jJlG zg31OWGhK{*Rgpp#`&~h0*Ght<4-;Xp4dIe0WW@;;*4X48o-k=8y(5e9k_v})QO$%C zhh-RNiGIv#?|~%e)IKm$8i?l}+2BeaYY01d^y-a>>zL;o$j{N70Y5uM_U#Z^I1{Ua zf%z$1gA3u9Fn>KVzVgfn@58W$mkwsmeOcH0eds&!b~L%Hz^c5KWFub8ffUu7ptNHc z9Q&FHJmqhw+Gio;Zwp1?T`efQM4GAGy&eieT%d2?TpHIR#P;*9ht0}s(OG2}hI~Q1ctSRdwR57)u}TFlgfxfgP)tIyJ3cqjHgY^0;@L2Pdo|hQR%pC9- z$H_@EqoW^VymAgW=l_I$&xBa7re4@$RZ9-4R-lueG-$2thk52I%(=~%;hCKb`{27WaI{nyHQ%F9PEXLn7()N{Q9i|ALU*Yu0GFZbb>M7%#G0W^ZHp>1Y4?mYL2Zm_P0xSz-A!rVQu zZD2o85b=N}YYyWs8ISX#D)8{rIXKh42&1D_S&`TsI5stwEUFlQvv*y>)n|lo<4{k0 znp;FeCG4QxeKpuPOvMwU#nJt}GE+QF6x?&dk1!|bE0K$^L1J_f zGL%jnqPZxm=k|6w?V|%69XdFNyTBIKFUtYZ zp-SwYWL=D^l%NK?p21g_17KbK5&eUTk<;`Xvf@5sNDhNT`N{iD#qBg?#BIzk&A~h4 z0*SVB7M{|(hgak4v1wBc92%v|-SI=1m0PwNWesa!Q(O~HQr>In*l2{W)D2ln?*a7V zpzC12C=sNbzhLsE(e&-pp=gk=$S$6v%3RExg@^X`<>zqI@wHSK@*-u~zGr_zplLao zFL?`vj|nlY&+pQ_kze65X#&^sQv7-4GAa)p#5gq`$M_b7^Vc_csj z0;+A2fG@GJDE2gvH?saRT#1uod>)>L7h@86mnZB4^TagpO?M)>&1vAKrOX~uwZk_a z7ty`sF2r4Yirc=dMXL`Y;1X~ht&FTlxJDk=q@X|1h{*@jm>QINCCbFR?#H8VKj7+F zg}6g{81%k25_ivS!Vjy{@n+lwa7lH8h58;?l-iE-(mcR!{7ndSKLO6J#~?XY4r<@o zlHn^#NxxA`A!KVm#+J3gJ^E4Tdqs;@+?!1AzLa8AuA4BGvmaqe_!is*a!mV#1NcUI zHfo${gD{Q|b7xEygr@1j1d|S~f?*5HHr$3&`7z7J?K~)|S%K)=BH|&mjOV_6A6E13 zLFUX4&}-snlpdLd9^-T1$I21xEmeKI?(l>>cD@B?Bm8kAn}!=?chI&~^Pps1DGqvU ziQh*~0Fx_9%-e`zxYsxgwuj0w8{2r~Qp{s$s=N+4%ongMZ-eP4HJP?=8K4&zg@zlh z!bOJ&DD!V8tG{TVn_L*|3^m1ZXWn4dw=nLY0xhnhc3;+LZ5D)WJ4e46><5dm37Ed9 z7rXsfFqp4kA^w0p-LKFKtAFUxw#ODIKS3LtCho>NoHn#pxs4^o&xnp}HY(M}k_}E0 z?9_g_VCZ@qEuReo#}jvOeeN;ro9ls%Sv-vND#FR;xQlAS2OymzWnwuaVbYZcFzd-)Zs4hGyu?vtoaAo6rKen&HR2--+#O4jO0LtReUq^EwGK0R zNfQ{M}s1b*l&&Y62;o1H2WQlG9KgRTB+76__i92iY+PygH z@^~eQpS1(87|OCq6StwqV_9~>D`TuwatGn5kEp%YUJT!p2=?Djqv5ipxVB^reX_C* zlhxOtTC_FQaUP8p38rLji#a$&6ymX0?%X*OjljosB2GFz9#WrWqglxl?tb*9@xR%+TMs4bqis*LnR*8=Hb&%q4+&wHeLI^3<~N)@oGyfq&kO_ll!XS!#E%Q z{Sr&|Ez^QY-&B~wtYCu+cAw#@{N%>knVyMahLIY}J6~XI;T%wpTMA8+aQxsy4z?y%apX)qt5izaQ}u+5nWFJG)n$gsXEk zz*6!YUKkMuOkfyYqOXq$&j!MTgJv+emBY5z4rcf_RO|)*Z(QL%N!aj^(gRcqLYKDV z+WTv8jgu@JAoYW$L{7p>^aqUiUJl3G6R}r`4<@ww!f>gRG`WStrYwkq>X~n_Vc87G zxTL|HOO6Mh%@auH_J;1wcl-@+g8htEU7H5}dXK^Lu5Av#wqgi+{9%v5RYSjpznCc~ z?S8EGw#CB|Oe6j_OSsw7kD}tRtE6I)7@XSG zi=;S3q5q=Z*wKE0rfpQn$r1~M-JxmRj7AU2s86DucJ8sK!|Io5{$;qowznPW(fkkg zj&M!+wV{YJ*7gOy)GEUZLleO3=OAimkO^#m4y2`Lq1yITxFwno2InQ%VGfgEij59c znXwYizHflx;}dveZ5mJl&hj?<%q8i<+QcSW46Af?KszoRPiwygSJ_~^xQM?V4*vv~ zKAoZ>7Xry?2L+yy+lrtEXgyQDH8MxVKFk_Q34#(^~NJbjFQOg;Uxa~!ZrNe?*|-t47VW@|eY9{}{Yh@Ke*iVMuSVChS-3Z1HR#FM;cj0Zy`vmKzrD)j zMXk^zx8?_7?TjRNUiJ*Ls+4JmMKLT9(&R}jvW0S?P!Kv60pHqlz_H^1e9hGY726>= z^_d$S&JG}#H-zA7HwotHoqaqfBE<9yEJDd}LsXw|7B*N~K%4qlXkF(<_dS$jjj9#k z+_O=Ktgzf}ojcD-nC?aoH&Oi}&9q{JeBd1LxaMi*h zSQQtEW((6v`KQ^*D^{>^eK|R%H3t=M)WYq}Ua*+^4&MA| zq;lCS$P3XL`gGD2^x1xZZW(d`r}sKU5*ECHlfH)h-`LgQz?%fMq4D&J^ccLC|CW|% zXTbf0d~&BCjP#R;!j)1Ukhp|_#G;Sj*RlYonw8?)csruvQ2?p49#jAEEUGS1XIfUA zCV4Az===t0ZW2gZ2QIu2b{ z&VwzH2QYj{{7Oe3Yh+KH|U(21X2rR@WL2xPJMq7mnBvV*Cx>}KVj=re%$lr;Epw0 z>1v+=FmAw5T<0W>5(bGhJ`evg(mF2ug&@XWqt@f8mJ3F6Fa!|RbS9&lMZ^ANwEEG4`BR+dE`#=LijYI78vuTcw^dg zj2u22ul9|>K{xYYL~XcdeHry`=HS{lQjFClF=j}lE3At-LYKSc0ki!w zRE0-?y!~TtjzR*ATV+Xa8tlMg&HcE~>Lvu`Rpczt@x>9>r0F)%BWPxE9=)kL_-@~T zQN|~6qga33D$U{TP}k-^zrF>Ych%78^jqvmmIK#>6__Luhps~&LD}cs5Rh<+)S6b} zAvtR@RZo?M_j`p^J_%5}GLN28IROc?JTUNa2C4dFMMCW=it4|#qea4F7sn)A@&LHi|nmM@Gj&v7K}bY4Bps*rEm_` zJ-JOxm+ip8bE=5)UT;vEQiRn@3dox;1Mt=G(fHNwCHZKm$m-?>kSv;Y(8@ae%>yj5{Fe7 zNBiD*B=sfF=xBDftm|dSAEgbuji2FtokZNSBOWqeUI*=`0bC7xAtsiF!xoIi1@ghD zoze^agROxTvd5$nYfKs!1!4!+!D6?*Se_IBiR&F8R>>FEjhVoEX43)l-rvRXhLJq| zgIUl%$eC;Fah!ZxBah^;FGRN8gSHxJZrO=(fJ0A!hS^DA+k&}!XQ$xNd^3D8@i7LY z7&|vl0kgskp?-!8bJOBBk+z;i!%iHt$DiaFFv8a(;;Ye7E<8MN{YW}4oDjQz5VWPX(8x}Q$4ICmBd zvphqz@^|1MnO-P9qd%03mVjxh5+nrf1bo!ao4flqG=9uPU7fKs(&rqVCl`aoSqs47 z<5Mc)X8^;+X!gdkVBF9?i5#=qK~LudVxhD+Hc5oit9zbP+m0XHIcL8SlfiXdE%X7g zWtVx!&F-My!|znpYB;9m6+yt}Thz1lIKZ^8)OUF{Ibv;wt6L=*BgKKp`T7{mhbz*1 z?Uh(OzBfvQWJ90iUab2T7LMQDhY}+w=tWwR(hmkyM*9RTZd9ReS|-q2AsW6PkY@(Y zVd?XkN;rPxK(aw+AxeZb!SiW{An05Wtn51tPwvTrX>WYM)B85gRzYZ~oq-{IzLzb6 zVTIQTZj{YHviV3kEIVlcuYb;<ej^VGmgD-QLcFls2u-eC zgLkg!SLHYMVO|XtWv`yDhPnYk$cMzutZNcad z1#s%A1-YO68AZhpgI&>JNbr~d*OaEix*-QatLikw^%+HiZtS3&cSd8#054eb@j5x7 zZw>Ee=5b@rxzX&A5pdRW93g>w;lh)dcu_bM^{lM$J*0s{bpn=tmc*R612Fz`C^#-W z4%2PyLB!YJGHB}_@K=9E+^(F4o8~FF^qvg{Y`#GY5A24$Nr2z=GErlE6}m2rqSp>A zfTZQ)@pxep)a_8E(?0aUB_mhkz4{|u%hYo?R&p^`IyTZDtNWAd=S;9BK9f|a z4{){g8YI}HfzhZ4)V|N+&_x+U!p)k$m$wLI_}7d}kP|%6szSY2Imon}hbBWi2){Um z_io^6oOS1Xgz(lm(bsLaCXm;0f_qkyE8 z{owCQ^`_pRC!@%$AbeHeLzq4HaL~H@&|Gc+hLextGNtF3-}gMvHD80*>t;H>YJCkk z>}A**Y6Gt7(PSL25d-?1rh$8NxWn$Brkso|?;s5izev7_hq-%quRyylebD5mH>VpZts*T-yL0<2_Mnr6YQ0 zw1b(hJl+0dDZYK*pUaKRgOeTgRHG`D)D^aq1ND30s>WU7?ID9vt)Y<8-x|WFJm!_j zN-@^c!qH%E9+`G<7&J}2L4_aAB=6?f;=|qL0J9!~+`4*9t6ELhI((&HLYKqBe&+nV zC*?^lb;Y@x{ji9492!@g$F?#3(RI%me06s#=AKh!6sixQ)bI3y8I&l5h!0CQPBzN5$m}yY~+0)nZuC!2WT+T(qC8>Dwiz7ZCHx`!r z-p2SSZ=4&u8Pwx;lC>XO!1nzl2z5!PVOqgxd@}&8{mS^yiqk>;(lVHM(-LnMeWM&uY3#G5~BfLs9Pj z1RC_(8{)<#!iL3D(A0N$j^ET=Qgy-s)+Nt{$tJg;Wbhp@dKQAh`j_eQzTpr$J`D$6 zD**A(CStJoFuHNH@Qsr@jo8&demY#DOg#Vj9JiU=iOAy?8+p^H*zZJn-E~-#qCzeX zEGL^VgmMjD@)}L&;~(=n-3f!g&NF0n9aHJfPW&maQ?~tJ=&x)}w`Ueyf&p)1Ke?^DaV^z4xr z>DE(lmQhgtTHlpj$7%e2wdMco?%oaaYj(&W^Iu?%j}01(y@3s{V<7#b3YqsfnR|Wu z5N2QUVo2?K7{BjTV9%aj4DJVNVcn*m=#|ulEB1e(?^~B*?#o?d&m|dVP&I#FWbij! z>;IHIKX(~ceuzc;ttE8e4pGK>lnKho&xA*VcqDq*T^tkl0N3$zd)KfpTX%5@{LwFe zZ=7~ruTqli9@ihnIdbUYE5FFHN$0pC(*k*&^1MVOIaX$Tg5cEDe|;8`f1>!`j+Gh- zm}m=d;{4N)q8yhs>%4sa?S*@S0SO5Q9?R$y|6d=P)93g9{gB+i9`GPO{3kc-MfMN= zcMJ26Jqwf(`E^FxSkR4Mx{nk8epYQTAOHJVwPAvWf3UE-ZoB+{A>#KlrGocU3Y6(S z1v&ITh*%<*Tl2s5+wVg%6!7m+{)XQsFBzHKDSq_79d;D%G@`=HPL`5<)AIlRXrzxy z$5j6dBmJ)QkRA&E0{`F#Z;uT_{zA)n-51MWXlcc6PXEP1NJmImaNw8I{l>}eTKdwbq|54?HzdBYy~QQgN+ zd*DZRzw@pK-lE$!|Bg2??)L1TrhQP|nO>dd_Ww&$$^SBy@VX60I`hyR@h;2;FSA*l ht`+B-nr}|=UoYC+;N6AgzqsMg Classifier + +Rebuild the fixed architecture and load `ps`/`st` from the JLD2 artifact at +`path`. Throws if the file is missing or unreadable — the server fails fast at +startup rather than run silently without classification. +""" +function load_classifier(path::AbstractString) + isfile(path) || error("model artifact not found at $path (run bin/train.jl to build it)") + data = JLD2.load(path) + (haskey(data, "ps") && haskey(data, "st")) || + error("model artifact at $path is missing ps/st (was it written by bin/train.jl?)") + return Classifier(build_model(), data["ps"], data["st"]) +end + +""" + classify(clf, path) -> Symbol + +Return `:known` or `:unknown` for the file at `path`. Files shorter than +`MIN_FILE_BYTES` short-circuit to `:unknown` without touching the model +(they can't form a 16+16 feature window). Otherwise: scale bytes, run the net, +argmax the two logits (class 1 = known, class 2 = unknown). +""" +function classify(clf::Classifier, path::AbstractString) + feats = read_features(path) + feats === nothing && return :unknown + x = reshape(feats, FEATURE_DIM, 1) + logits, _ = Lux.apply(clf.model, x, clf.ps, clf.st) + return argmax(vec(logits)) == 1 ? :known : :unknown +end diff --git a/src/config.jl b/src/config.jl index 6e4a237..46e2bdc 100644 --- a/src/config.jl +++ b/src/config.jl @@ -10,6 +10,7 @@ Base.@kwdef struct Config spool_dir::String = "data/spool" # files land here on intake (pending) done_dir::String = "data/done" # files move here after successful processing failed_dir::String = "data/failed" # files move here if a worker throws + model_path::String = "model/classifier.jld2" # committed classifier artifact, loaded at startup end """ @@ -21,11 +22,11 @@ and for `FileServer.run(; port=...)`). Recognised variables: FS_HOST, FS_PORT, FS_WORKERS, FS_QUEUE_CAPACITY, - FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR + FS_SPOOL_DIR, FS_DONE_DIR, FS_FAILED_DIR, FS_MODEL_PATH """ function config_from_env(; host=nothing, port=nothing, worker_count=nothing, queue_capacity=nothing, spool_dir=nothing, - done_dir=nothing, failed_dir=nothing) + done_dir=nothing, failed_dir=nothing, model_path=nothing) Config( host = something(host, get(ENV, "FS_HOST", "127.0.0.1")), port = something(port, parse(Int, get(ENV, "FS_PORT", "8080"))), @@ -34,6 +35,7 @@ function config_from_env(; host=nothing, port=nothing, worker_count=nothing, spool_dir = something(spool_dir, get(ENV, "FS_SPOOL_DIR", "data/spool")), done_dir = something(done_dir, get(ENV, "FS_DONE_DIR", "data/done")), failed_dir = something(failed_dir, get(ENV, "FS_FAILED_DIR", "data/failed")), + model_path = something(model_path, get(ENV, "FS_MODEL_PATH", "model/classifier.jld2")), ) end diff --git a/src/model.jl b/src/model.jl new file mode 100644 index 0000000..e0f7baa --- /dev/null +++ b/src/model.jl @@ -0,0 +1,51 @@ +# Shared model definition + feature extraction, used by BOTH the training script +# (bin/train.jl) and the server (worker inference). Keeping the architecture and +# the byte->feature mapping in one place guarantees train and inference can never +# drift out of sync (same input layout, same scaling, same net shape). +# +# The including scope is expected to have already done `using Lux` and +# `using JLD2` (FileServer.jl and bin/train.jl both do), so this file adds no +# top-level `using` of its own and works whether it's pulled into the FileServer +# module or a bare script. + +"Number of input features: first 16 bytes + last 16 bytes of a file." +const FEATURE_DIM = 32 + +"Minimum file size (bytes) the model can accept: needs a disjoint 16+16 window." +const MIN_FILE_BYTES = 32 + +""" + build_model() + +The fixed-structure classifier: 32-dim byte features → two logits +(class 1 = known, class 2 = unknown). Raw logits out (no softmax layer); the +loss applies log-softmax and inference just takes an argmax. +""" +build_model() = Chain( + Dense(FEATURE_DIM => 64, relu), + Dense(64 => 16, relu), + Dense(16 => 2), +) + +""" + read_features(path) -> Union{Vector{Float32}, Nothing} + +Read the first 16 and last 16 bytes of the file at `path`, concatenate to a +32-element vector, and scale each byte from 0–255 into [0,1]. Returns `nothing` +for files shorter than `MIN_FILE_BYTES` (the caller decides what that means: +training drops them, the server classifies them as unknown without the model). + +Memory stays flat: we read 16 bytes from the front and `seek` to end-16 for the +tail rather than slurping the whole file. +""" +function read_features(path::AbstractString) + sz = filesize(path) + sz < MIN_FILE_BYTES && return nothing + bytes = open(path) do io + head = read(io, 16) + seek(io, sz - 16) + tail = read(io, 16) + vcat(head, tail) + end + return Float32.(bytes) ./ 255f0 +end diff --git a/src/worker.jl b/src/worker.jl index 289b930..f39dc71 100644 --- a/src/worker.jl +++ b/src/worker.jl @@ -10,9 +10,11 @@ For now the "work" is just logging the received filename to prove the flow — this is the seam where real heavy-lifting will go later. """ function handle_job(job::Job, cfg::Config, worker_id::Int) - # --- placeholder for real heavy-lifting work ------------------------- - @info "received file" worker=worker_id id=job.id name=job.original_name size=job.size - # -------------------------------------------------------------------- + # Classify the spooled file (annotate-only for now: the result is logged but + # every file still moves to done/ regardless of known/unknown). Sub-32-byte + # files short-circuit to :unknown inside classify without touching the model. + classification = classify(CLASSIFIER[], job.path) + @info "received file" worker=worker_id id=job.id name=job.original_name size=job.size classification=classification dest = move_to(cfg.done_dir, job) @info "completed" worker=worker_id id=job.id dest=dest return nothing