1731 lines
48 KiB
Julia
1731 lines
48 KiB
Julia
# This file is a part of Julia. License is MIT: https://julialang.org/license
|
|
|
|
using Base.Test
|
|
include("testenv.jl")
|
|
|
|
# Test a few "remote" invocations when no workers are present
|
|
@test remote(myid)() == 1
|
|
@test pmap(identity, 1:100) == [1:100...]
|
|
@test 100 == @parallel (+) for i in 1:100
|
|
1
|
|
end
|
|
|
|
addprocs_with_testenv(4)
|
|
@test nprocs() == 5
|
|
|
|
function reuseport_tests()
|
|
# Run the test on all processes.
|
|
results = asyncmap(procs()) do p
|
|
remotecall_fetch(p) do
|
|
ports_lower = [] # ports of pids lower than myid()
|
|
ports_higher = [] # ports of pids higher than myid()
|
|
for w in Base.Distributed.PGRP.workers
|
|
w.id == myid() && continue
|
|
port = Base._sockname(w.r_stream, true)[2]
|
|
if (w.id == 1)
|
|
# master connects to workers
|
|
push!(ports_higher, port)
|
|
elseif w.id < myid()
|
|
push!(ports_lower, port)
|
|
elseif w.id > myid()
|
|
push!(ports_higher, port)
|
|
end
|
|
end
|
|
@assert (length(ports_lower) + length(ports_higher)) == nworkers()
|
|
for portset in [ports_lower, ports_higher]
|
|
if (length(portset) > 0) && (length(unique(portset)) != 1)
|
|
warn("SO_REUSEPORT TESTS FAILED. UNSUPPORTED/OLDER UNIX VERSION?")
|
|
return 0
|
|
end
|
|
end
|
|
return myid()
|
|
end
|
|
end
|
|
|
|
# Ensure that the code has indeed been successfully executed everywhere
|
|
@test all(p -> p in results, procs())
|
|
end
|
|
|
|
# Test that the client port is reused. SO_REUSEPORT may not be supported on
|
|
# all UNIX platforms, Linux kernels prior to 3.9 and older versions of OSX
|
|
if is_unix()
|
|
# Run reuse client port tests only if SO_REUSEPORT is supported.
|
|
s = TCPSocket(delay = false)
|
|
is_linux() && Base.Distributed.bind_client_port(s)
|
|
if ccall(:jl_tcp_reuseport, Int32, (Ptr{Void},), s.handle) == 0
|
|
# Client reuse port has been disabled in 0.6.2
|
|
# reuseport_tests()
|
|
else
|
|
info("SO_REUSEPORT is unsupported, skipping reuseport tests.")
|
|
end
|
|
end
|
|
|
|
id_me = myid()
|
|
id_other = filter(x -> x != id_me, procs())[rand(1:(nprocs()-1))]
|
|
|
|
# Test remote()
|
|
let
|
|
pool = Base.default_worker_pool()
|
|
|
|
count = 0
|
|
count_condition = Condition()
|
|
|
|
function remote_wait(c)
|
|
@async begin
|
|
count += 1
|
|
remote(take!)(c)
|
|
count -= 1
|
|
notify(count_condition)
|
|
end
|
|
yield()
|
|
end
|
|
|
|
testchannels = [RemoteChannel() for i in 1:nworkers()]
|
|
testcount = 0
|
|
@test isready(pool) == true
|
|
for c in testchannels
|
|
@test count == testcount
|
|
remote_wait(c)
|
|
testcount += 1
|
|
end
|
|
@test count == testcount
|
|
@test isready(pool) == false
|
|
|
|
for c in testchannels
|
|
@test count == testcount
|
|
put!(c, "foo")
|
|
testcount -= 1
|
|
wait(count_condition)
|
|
@test count == testcount
|
|
@test isready(pool) == true
|
|
end
|
|
|
|
@test count == 0
|
|
|
|
for c in testchannels
|
|
@test count == testcount
|
|
remote_wait(c)
|
|
testcount += 1
|
|
end
|
|
@test count == testcount
|
|
@test isready(pool) == false
|
|
|
|
for c in reverse(testchannels)
|
|
@test count == testcount
|
|
put!(c, "foo")
|
|
testcount -= 1
|
|
wait(count_condition)
|
|
@test count == testcount
|
|
@test isready(pool) == true
|
|
end
|
|
|
|
@test count == 0
|
|
end
|
|
|
|
# Test Futures
|
|
function testf(id)
|
|
f=Future(id)
|
|
@test isready(f) == false
|
|
@test isnull(f.v) == true
|
|
put!(f, :OK)
|
|
@test isready(f) == true
|
|
@test isnull(f.v) == false
|
|
|
|
@test_throws ErrorException put!(f, :OK) # Cannot put! to a already set future
|
|
@test_throws MethodError take!(f) # take! is unsupported on a Future
|
|
|
|
@test fetch(f) == :OK
|
|
end
|
|
|
|
testf(id_me)
|
|
testf(id_other)
|
|
|
|
# Distributed GC tests for Futures
|
|
function test_futures_dgc(id)
|
|
f = remotecall(myid, id)
|
|
fid = Base.remoteref_id(f)
|
|
|
|
# remote value should be deleted after a fetch
|
|
@test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, fid) == true
|
|
@test isnull(f.v) == true
|
|
@test fetch(f) == id
|
|
@test isnull(f.v) == false
|
|
yield(); # flush gc msgs
|
|
@test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, fid) == false
|
|
|
|
|
|
# if unfetched, it should be deleted after a finalize
|
|
f = remotecall(myid, id)
|
|
fid = Base.remoteref_id(f)
|
|
@test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, fid) == true
|
|
@test isnull(f.v) == true
|
|
finalize(f)
|
|
yield(); # flush gc msgs
|
|
@test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, fid) == false
|
|
end
|
|
|
|
test_futures_dgc(id_me)
|
|
test_futures_dgc(id_other)
|
|
|
|
# if sent to another worker, it should not be deleted till all references are fetched.
|
|
wid1 = workers()[1]
|
|
wid2 = workers()[2]
|
|
f = remotecall(myid, wid1)
|
|
fid = Base.remoteref_id(f)
|
|
|
|
fstore = RemoteChannel(wid2)
|
|
put!(fstore, f)
|
|
|
|
@test fetch(f) == wid1
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == true
|
|
remotecall_fetch(r->(fetch(fetch(r)); yield()), wid2, fstore)
|
|
sleep(0.5) # to ensure that wid2 gc messages have been executed on wid1
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == false
|
|
|
|
# put! should release remote reference since it would have been cached locally
|
|
f = Future(wid1)
|
|
fid = Base.remoteref_id(f)
|
|
|
|
# should not be created remotely till accessed
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == false
|
|
# create it remotely
|
|
isready(f)
|
|
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == true
|
|
put!(f, :OK)
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == false
|
|
@test fetch(f) == :OK
|
|
|
|
# RemoteException should be thrown on a put! when another process has set the value
|
|
f = Future(wid1)
|
|
fid = Base.remoteref_id(f)
|
|
|
|
fstore = RemoteChannel(wid2)
|
|
put!(fstore, f) # send f to wid2
|
|
put!(f, :OK) # set value from master
|
|
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == true
|
|
|
|
testval = remotecall_fetch(wid2, fstore) do x
|
|
try
|
|
put!(fetch(x), :OK)
|
|
return 0
|
|
catch e
|
|
if isa(e, RemoteException)
|
|
return 1
|
|
else
|
|
return 2
|
|
end
|
|
end
|
|
end
|
|
@test testval == 1
|
|
|
|
# Distributed GC tests for RemoteChannels
|
|
function test_remoteref_dgc(id)
|
|
rr = RemoteChannel(id)
|
|
put!(rr, :OK)
|
|
rrid = Base.remoteref_id(rr)
|
|
|
|
# remote value should be deleted after finalizing the ref
|
|
@test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, rrid) == true
|
|
@test fetch(rr) == :OK
|
|
@test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, rrid) == true
|
|
finalize(rr)
|
|
yield(); # flush gc msgs
|
|
@test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, rrid) == false
|
|
end
|
|
test_remoteref_dgc(id_me)
|
|
test_remoteref_dgc(id_other)
|
|
|
|
# if sent to another worker, it should not be deleted till the other worker has also finalized.
|
|
wid1 = workers()[1]
|
|
wid2 = workers()[2]
|
|
rr = RemoteChannel(wid1)
|
|
rrid = Base.remoteref_id(rr)
|
|
|
|
fstore = RemoteChannel(wid2)
|
|
put!(fstore, rr)
|
|
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, rrid) == true
|
|
finalize(rr) # finalize locally
|
|
yield(); # flush gc msgs
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, rrid) == true
|
|
remotecall_fetch(r->(finalize(take!(r)); yield(); nothing), wid2, fstore) # finalize remotely
|
|
sleep(0.5) # to ensure that wid2 messages have been executed on wid1
|
|
@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, rrid) == false
|
|
|
|
# Tests for issue #23109 - should not hang.
|
|
f = @spawn rand(1,1)
|
|
@sync begin
|
|
for _ in 1:10
|
|
@async fetch(f)
|
|
end
|
|
end
|
|
|
|
wid1,wid2 = workers()[1:2]
|
|
f = @spawnat wid1 rand(1,1)
|
|
@sync begin
|
|
@async fetch(f)
|
|
@async remotecall_fetch(()->fetch(f), wid2)
|
|
end
|
|
|
|
|
|
@test fetch(@spawnat id_other myid()) == id_other
|
|
@test (@fetchfrom id_other myid()) == id_other
|
|
|
|
pids=[]
|
|
for i in 1:nworkers()
|
|
push!(pids, @fetch myid())
|
|
end
|
|
@test sort(pids) == sort(workers())
|
|
|
|
|
|
# test getindex on Futures and RemoteChannels
|
|
function test_indexing(rr)
|
|
a = rand(5,5)
|
|
put!(rr, a)
|
|
@test rr[2,3] == a[2,3]
|
|
@test rr[] == a
|
|
end
|
|
|
|
test_indexing(Future())
|
|
test_indexing(Future(id_other))
|
|
test_indexing(RemoteChannel())
|
|
test_indexing(RemoteChannel(id_other))
|
|
|
|
dims = (20,20,20)
|
|
|
|
if is_linux()
|
|
S = SharedArray{Int64,3}(dims)
|
|
@test startswith(S.segname, "/jl")
|
|
@test !ispath("/dev/shm" * S.segname)
|
|
|
|
S = SharedArray{Int64,3}(dims; pids=[id_other])
|
|
@test startswith(S.segname, "/jl")
|
|
@test !ispath("/dev/shm" * S.segname)
|
|
end
|
|
|
|
# TODO : Need a similar test of shmem cleanup for OSX
|
|
|
|
##### SharedArray tests
|
|
|
|
function check_pids_all(S::SharedArray)
|
|
pidtested = falses(size(S))
|
|
for p in procs(S)
|
|
idxes_in_p = remotecall_fetch(p, S) do D
|
|
parentindexes(D.loc_subarr_1d)[1]
|
|
end
|
|
@test all(sdata(S)[idxes_in_p] .== p)
|
|
pidtested[idxes_in_p] = true
|
|
end
|
|
@test all(pidtested)
|
|
end
|
|
|
|
d = Base.shmem_rand(1:100, dims)
|
|
a = convert(Array, d)
|
|
|
|
partsums = Array{Int}(length(procs(d)))
|
|
@sync begin
|
|
for (i, p) in enumerate(procs(d))
|
|
@async partsums[i] = remotecall_fetch(p, d) do D
|
|
sum(D.loc_subarr_1d)
|
|
end
|
|
end
|
|
end
|
|
@test sum(a) == sum(partsums)
|
|
|
|
d = Base.shmem_rand(dims)
|
|
for p in procs(d)
|
|
idxes_in_p = remotecall_fetch(p, d) do D
|
|
parentindexes(D.loc_subarr_1d)[1]
|
|
end
|
|
idxf = first(idxes_in_p)
|
|
idxl = last(idxes_in_p)
|
|
d[idxf] = Float64(idxf)
|
|
rv = remotecall_fetch(p, d,idxf,idxl) do D,idxf,idxl
|
|
assert(D[idxf] == Float64(idxf))
|
|
D[idxl] = Float64(idxl)
|
|
D[idxl]
|
|
end
|
|
@test d[idxl] == rv
|
|
end
|
|
|
|
@test ones(10, 10, 10) == Base.shmem_fill(1.0, (10,10,10))
|
|
@test zeros(Int32, 10, 10, 10) == Base.shmem_fill(0, (10,10,10))
|
|
|
|
d = Base.shmem_rand(dims)
|
|
s = Base.shmem_rand(dims)
|
|
copy!(s, d)
|
|
@test s == d
|
|
s = Base.shmem_rand(dims)
|
|
copy!(s, sdata(d))
|
|
@test s == d
|
|
a = rand(dims)
|
|
@test sdata(a) == a
|
|
|
|
d = SharedArray{Int}(dims, init = D->fill!(D.loc_subarr_1d, myid()))
|
|
for p in procs(d)
|
|
idxes_in_p = remotecall_fetch(p, d) do D
|
|
parentindexes(D.loc_subarr_1d)[1]
|
|
end
|
|
idxf = first(idxes_in_p)
|
|
idxl = last(idxes_in_p)
|
|
@test d[idxf] == p
|
|
@test d[idxl] == p
|
|
end
|
|
|
|
d = @inferred(SharedArray{Float64,2}((2,3)))
|
|
@test isa(d[:,2], Vector{Float64})
|
|
|
|
### SharedArrays from a file
|
|
|
|
# Mapping an existing file
|
|
fn = tempname()
|
|
write(fn, 1:30)
|
|
sz = (6,5)
|
|
Atrue = reshape(1:30, sz)
|
|
|
|
S = @inferred(SharedArray{Int,2}(fn, sz))
|
|
@test S == Atrue
|
|
@test length(procs(S)) > 1
|
|
@sync begin
|
|
for p in procs(S)
|
|
@async remotecall_wait(p, S) do D
|
|
fill!(D.loc_subarr_1d, myid())
|
|
end
|
|
end
|
|
end
|
|
check_pids_all(S)
|
|
|
|
filedata = similar(Atrue)
|
|
read!(fn, filedata)
|
|
@test filedata == sdata(S)
|
|
finalize(S)
|
|
|
|
# Error for write-only files
|
|
@test_throws ArgumentError SharedArray{Int,2}(fn, sz, mode="w")
|
|
|
|
# Error for file doesn't exist, but not allowed to create
|
|
@test_throws ArgumentError SharedArray{Int,2}(joinpath(tempdir(),randstring()), sz, mode="r")
|
|
|
|
# Creating a new file
|
|
fn2 = tempname()
|
|
S = SharedArray{Int,2}(fn2, sz, init=D->D[localindexes(D)] = myid())
|
|
@test S == filedata
|
|
filedata2 = similar(Atrue)
|
|
read!(fn2, filedata2)
|
|
@test filedata == filedata2
|
|
finalize(S)
|
|
|
|
# Appending to a file
|
|
fn3 = tempname()
|
|
write(fn3, ones(UInt8, 4))
|
|
S = SharedArray{UInt8}(fn3, sz, 4, mode="a+", init=D->D[localindexes(D)]=0x02)
|
|
len = prod(sz)+4
|
|
@test filesize(fn3) == len
|
|
filedata = Array{UInt8}(len)
|
|
read!(fn3, filedata)
|
|
@test all(filedata[1:4] .== 0x01)
|
|
@test all(filedata[5:end] .== 0x02)
|
|
finalize(S)
|
|
|
|
# call gc 3 times to avoid unlink: operation not permitted (EPERM) on Windows
|
|
S = nothing
|
|
@everywhere gc()
|
|
@everywhere gc()
|
|
@everywhere gc()
|
|
rm(fn); rm(fn2); rm(fn3)
|
|
|
|
### Utility functions
|
|
|
|
# construct PR #13514
|
|
S = @inferred(SharedArray{Int}((1,2,3)))
|
|
@test size(S) == (1,2,3)
|
|
@test typeof(S) <: SharedArray{Int}
|
|
S = @inferred(SharedArray{Int}(2))
|
|
@test size(S) == (2,)
|
|
@test typeof(S) <: SharedArray{Int}
|
|
S = @inferred(SharedArray{Int}(1,2))
|
|
@test size(S) == (1,2)
|
|
@test typeof(S) <: SharedArray{Int}
|
|
S = @inferred(SharedArray{Int}(1,2,3))
|
|
@test size(S) == (1,2,3)
|
|
@test typeof(S) <: SharedArray{Int}
|
|
|
|
# reshape
|
|
|
|
d = Base.shmem_fill(1.0, (10,10,10))
|
|
@test ones(100, 10) == reshape(d,(100,10))
|
|
d = Base.shmem_fill(1.0, (10,10,10))
|
|
@test_throws DimensionMismatch reshape(d,(50,))
|
|
|
|
# rand, randn
|
|
d = Base.shmem_rand(dims)
|
|
@test size(rand!(d)) == dims
|
|
d = Base.shmem_fill(1.0, dims)
|
|
@test size(randn!(d)) == dims
|
|
|
|
# similar
|
|
d = Base.shmem_rand(dims)
|
|
@test size(similar(d, Complex128)) == dims
|
|
@test size(similar(d, dims)) == dims
|
|
|
|
# issue #6362
|
|
d = Base.shmem_rand(dims)
|
|
s = copy(sdata(d))
|
|
ds = deepcopy(d)
|
|
@test ds == d
|
|
pids_d = procs(d)
|
|
remotecall_fetch(setindex!, pids_d[findfirst(id->(id != myid()), pids_d)], d, 1.0, 1:10)
|
|
@test ds != d
|
|
@test s != d
|
|
copy!(d, s)
|
|
@everywhere setid!(A) = A[localindexes(A)] = myid()
|
|
@sync for p in procs(ds)
|
|
@async remotecall_wait(setid!, p, ds)
|
|
end
|
|
@test d == s
|
|
@test ds != s
|
|
@test first(ds) == first(procs(ds))
|
|
@test last(ds) == last(procs(ds))
|
|
|
|
|
|
# SharedArray as an array
|
|
# Since the data in d will depend on the nprocs, just test that these operations work
|
|
a = d[1:5]
|
|
@test_throws BoundsError d[-1:5]
|
|
a = d[1,1,1:3:end]
|
|
d[2:4] = 7
|
|
d[5,1:2:4,8] = 19
|
|
|
|
AA = rand(4,2)
|
|
A = @inferred(convert(SharedArray, AA))
|
|
B = @inferred(convert(SharedArray, AA'))
|
|
@test B*A == ctranspose(AA)*AA
|
|
|
|
d=SharedArray{Int64,2}((10,10); init = D->fill!(D.loc_subarr_1d, myid()), pids=[id_me, id_other])
|
|
d2 = map(x->1, d)
|
|
@test reduce(+, d2) == 100
|
|
|
|
@test reduce(+, d) == ((50*id_me) + (50*id_other))
|
|
map!(x->1, d, d)
|
|
@test reduce(+, d) == 100
|
|
|
|
@test fill!(d, 1) == ones(10, 10)
|
|
@test fill!(d, 2.) == fill(2, 10, 10)
|
|
@test d[:] == fill(2, 100)
|
|
@test d[:,1] == fill(2, 10)
|
|
@test d[1,:] == fill(2, 10)
|
|
|
|
# Boundary cases where length(S) <= length(pids)
|
|
@test 2.0 == remotecall_fetch(D->D[2], id_other, Base.shmem_fill(2.0, 2; pids=[id_me, id_other]))
|
|
@test 3.0 == remotecall_fetch(D->D[1], id_other, Base.shmem_fill(3.0, 1; pids=[id_me, id_other]))
|
|
|
|
# Shared arrays of singleton immutables
|
|
@everywhere struct ShmemFoo end
|
|
for T in [Void, ShmemFoo]
|
|
s = @inferred(SharedArray{T}(10))
|
|
@test T() === remotecall_fetch(x->x[3], workers()[1], s)
|
|
end
|
|
|
|
# Issue #14664
|
|
d = SharedArray{Int}(10)
|
|
@sync @parallel for i=1:10
|
|
d[i] = i
|
|
end
|
|
|
|
for (x,i) in enumerate(d)
|
|
@test x == i
|
|
end
|
|
|
|
# complex
|
|
sd = SharedArray{Int}(10)
|
|
se = SharedArray{Int}(10)
|
|
@sync @parallel for i=1:10
|
|
sd[i] = i
|
|
se[i] = i
|
|
end
|
|
sc = convert(SharedArray, complex.(sd,se))
|
|
for (x,i) in enumerate(sc)
|
|
@test i == complex(x,x)
|
|
end
|
|
|
|
# Once finalized accessing remote references and shared arrays should result in exceptions.
|
|
function finalize_and_test(r)
|
|
finalize(r)
|
|
@test_throws ErrorException fetch(r)
|
|
end
|
|
|
|
for id in [id_me, id_other]
|
|
finalize_and_test(Future(id))
|
|
finalize_and_test((r=Future(id); put!(r, 1); r))
|
|
finalize_and_test(RemoteChannel(id))
|
|
finalize_and_test((r=RemoteChannel(id); put!(r, 1); r))
|
|
end
|
|
|
|
d = SharedArray{Int}(10)
|
|
finalize(d)
|
|
@test_throws BoundsError d[1]
|
|
|
|
# Issue 22139
|
|
aorig = a1 = SharedArray{Float64}((3, 3))
|
|
a1 = remotecall_fetch(fill!, id_other, a1, 1.0)
|
|
@test object_id(aorig) == object_id(a1)
|
|
id = a1.id
|
|
aorig = nothing
|
|
a1 = remotecall_fetch(fill!, id_other, a1, 1.0)
|
|
gc(); gc()
|
|
a1 = remotecall_fetch(fill!, id_other, a1, 1.0)
|
|
@test haskey(Base.sa_refs, id)
|
|
finalize(a1)
|
|
@test !haskey(Base.sa_refs, id)
|
|
|
|
# Test @parallel load balancing - all processors should get either M or M+1
|
|
# iterations out of the loop range for some M.
|
|
ids = @parallel((a,b)->[a;b], for i=1:7; myid(); end)
|
|
workloads = Int[sum(ids .== i) for i in 2:nprocs()]
|
|
@test maximum(workloads) - minimum(workloads) <= 1
|
|
|
|
# @parallel reduction should work even with very short ranges
|
|
@test @parallel(+, for i=1:2; i; end) == 3
|
|
|
|
@test_throws ArgumentError sleep(-1)
|
|
@test_throws ArgumentError timedwait(()->false, 0.1, pollint=-0.5)
|
|
|
|
# specify pids for pmap
|
|
@test sort(workers()[1:2]) == sort(unique(pmap(WorkerPool(workers()[1:2]), x->(sleep(0.1);myid()), 1:10)))
|
|
|
|
# Testing buffered and unbuffered reads
|
|
# This large array should write directly to the socket
|
|
a = ones(10^6)
|
|
@test a == remotecall_fetch((x)->x, id_other, a)
|
|
|
|
# Not a bitstype, should be buffered
|
|
s = [randstring() for x in 1:10^5]
|
|
@test s == remotecall_fetch((x)->x, id_other, s)
|
|
|
|
#large number of small requests
|
|
num_small_requests = 10000
|
|
@test fill(id_other, num_small_requests) == [remotecall_fetch(myid, id_other) for i in 1:num_small_requests]
|
|
|
|
# test parallel sends of large arrays from multiple tasks to the same remote worker
|
|
ntasks = 10
|
|
rr_list = [Channel(1) for x in 1:ntasks]
|
|
|
|
for rr in rr_list
|
|
let rr=rr
|
|
@async try
|
|
for i in 1:10
|
|
a = rand(2*10^5)
|
|
@test a == remotecall_fetch(x->x, id_other, a)
|
|
yield()
|
|
end
|
|
put!(rr, :OK)
|
|
catch
|
|
put!(rr, :ERROR)
|
|
end
|
|
end
|
|
end
|
|
|
|
@test [fetch(rr) for rr in rr_list] == [:OK for x in 1:ntasks]
|
|
|
|
function test_channel(c)
|
|
put!(c, 1)
|
|
put!(c, "Hello")
|
|
put!(c, 5.0)
|
|
|
|
@test isready(c) == true
|
|
@test fetch(c) == 1
|
|
@test fetch(c) == 1 # Should not have been popped previously
|
|
@test take!(c) == 1
|
|
@test take!(c) == "Hello"
|
|
@test fetch(c) == 5.0
|
|
@test take!(c) == 5.0
|
|
@test isready(c) == false
|
|
close(c)
|
|
end
|
|
|
|
test_channel(Channel(10))
|
|
test_channel(RemoteChannel(()->Channel(10)))
|
|
|
|
c=Channel{Int}(1)
|
|
@test_throws MethodError put!(c, "Hello")
|
|
|
|
# test channel iterations
|
|
function test_iteration(in_c, out_c)
|
|
t=@schedule for v in in_c
|
|
put!(out_c, v)
|
|
end
|
|
|
|
isa(in_c, Channel) && @test isopen(in_c) == true
|
|
put!(in_c, 1)
|
|
@test take!(out_c) == 1
|
|
put!(in_c, "Hello")
|
|
close(in_c)
|
|
@test take!(out_c) == "Hello"
|
|
isa(in_c, Channel) && @test isopen(in_c) == false
|
|
@test_throws InvalidStateException put!(in_c, :foo)
|
|
yield()
|
|
@test istaskdone(t) == true
|
|
end
|
|
|
|
test_iteration(Channel(10), Channel(10))
|
|
# make sure exceptions propagate when waiting on Tasks
|
|
@test_throws CompositeException (@sync (@async error("oops")))
|
|
try
|
|
@sync begin
|
|
for i in 1:5
|
|
@async error(i)
|
|
end
|
|
end
|
|
error("unexpected")
|
|
catch ex
|
|
@test typeof(ex) == CompositeException
|
|
@test length(ex) == 5
|
|
@test typeof(ex.exceptions[1]) == CapturedException
|
|
@test typeof(ex.exceptions[1].ex) == ErrorException
|
|
errors = map(x->x.ex.msg, ex.exceptions)
|
|
@test collect(1:5) == sort(map(x->parse(Int, x), errors))
|
|
end
|
|
|
|
function test_remoteexception_thrown(expr)
|
|
try
|
|
expr()
|
|
error("unexpected")
|
|
catch ex
|
|
@test typeof(ex) == RemoteException
|
|
@test typeof(ex.captured) == CapturedException
|
|
@test typeof(ex.captured.ex) == ErrorException
|
|
@test ex.captured.ex.msg == "foobar"
|
|
end
|
|
end
|
|
|
|
for id in [id_other, id_me]
|
|
test_remoteexception_thrown() do
|
|
remotecall_fetch(id) do
|
|
throw(ErrorException("foobar"))
|
|
end
|
|
end
|
|
test_remoteexception_thrown() do
|
|
remotecall_wait(id) do
|
|
throw(ErrorException("foobar"))
|
|
end
|
|
end
|
|
test_remoteexception_thrown() do
|
|
wait(remotecall(id) do
|
|
throw(ErrorException("foobar"))
|
|
end)
|
|
end
|
|
end
|
|
|
|
# make sure the stackframe from the remote error can be serialized
|
|
let ex
|
|
try
|
|
remotecall_fetch(id_other) do
|
|
@eval module AModuleLocalToOther
|
|
foo() = throw(ErrorException("A.error"))
|
|
foo()
|
|
end
|
|
end
|
|
catch ex
|
|
end
|
|
@test (ex::RemoteException).pid == id_other
|
|
@test ((ex.captured::CapturedException).ex::ErrorException).msg == "A.error"
|
|
bt = ex.captured.processed_bt::Array{Any,1}
|
|
@test length(bt) > 1
|
|
frame, repeated = bt[1]::Tuple{StackFrame, Int}
|
|
@test frame.func == :foo
|
|
@test isnull(frame.linfo)
|
|
@test repeated == 1
|
|
end
|
|
|
|
# pmap tests. Needs at least 4 processors dedicated to the below tests. Which we currently have
|
|
# since the distributed tests are now spawned as a separate set.
|
|
|
|
# Test all combinations of pmap keyword args.
|
|
pmap_args = [
|
|
(:distributed, [:default, false]),
|
|
(:batch_size, [:default,2]),
|
|
(:on_error, [:default, e -> (e.msg == "foobar" ? true : rethrow(e))]),
|
|
(:retry_delays, [:default, fill(0.001, 1000)]),
|
|
(:retry_check, [:default, (s,e) -> (s,endswith(e.msg,"foobar"))]),
|
|
]
|
|
|
|
kwdict = Dict()
|
|
function walk_args(i)
|
|
if i > length(pmap_args)
|
|
kwargs = []
|
|
for (k,v) in kwdict
|
|
if v !== :default
|
|
push!(kwargs, (k,v))
|
|
end
|
|
end
|
|
|
|
data = 1:100
|
|
|
|
testw = kwdict[:distributed] === false ? [1] : workers()
|
|
|
|
if kwdict[:retry_delays] !== :default
|
|
mapf = x -> iseven(myid()) ? error("notfoobar") : (x*2, myid())
|
|
results_test = pmap_res -> begin
|
|
results = [x[1] for x in pmap_res]
|
|
pids = [x[2] for x in pmap_res]
|
|
@test results == [2:2:200...]
|
|
for p in testw
|
|
if isodd(p)
|
|
@test p in pids
|
|
else
|
|
@test !(p in pids)
|
|
end
|
|
end
|
|
end
|
|
elseif kwdict[:on_error] === :default
|
|
mapf = x -> (x*2, myid())
|
|
results_test = pmap_res -> begin
|
|
results = [x[1] for x in pmap_res]
|
|
pids = [x[2] for x in pmap_res]
|
|
@test results == [2:2:200...]
|
|
for p in testw
|
|
@test p in pids
|
|
end
|
|
end
|
|
else
|
|
mapf = x -> iseven(x) ? error("foobar") : (x*2, myid())
|
|
results_test = pmap_res -> begin
|
|
w = testw
|
|
for (idx,x) in enumerate(data)
|
|
if iseven(x)
|
|
@test pmap_res[idx] == true
|
|
else
|
|
@test pmap_res[idx][1] == x*2
|
|
@test pmap_res[idx][2] in w
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
try
|
|
results_test(pmap(mapf, data; kwargs...))
|
|
catch e
|
|
println("pmap executing with args : ", kwargs)
|
|
rethrow(e)
|
|
end
|
|
|
|
return
|
|
end
|
|
|
|
kwdict[pmap_args[i][1]] = pmap_args[i][2][1]
|
|
walk_args(i+1)
|
|
|
|
kwdict[pmap_args[i][1]] = pmap_args[i][2][2]
|
|
walk_args(i+1)
|
|
end
|
|
|
|
# Start test for various kw arg combinations
|
|
walk_args(1)
|
|
|
|
# Simple test for pmap throws error
|
|
error_thrown = false
|
|
try
|
|
pmap(x -> x==50 ? error("foobar") : x, 1:100)
|
|
catch e
|
|
@test e.captured.ex.msg == "foobar"
|
|
error_thrown = true
|
|
end
|
|
@test error_thrown
|
|
|
|
# Test pmap with a generator type iterator
|
|
@test [1:100...] == pmap(x->x, Base.Generator(x->(sleep(0.0001); x), 1:100))
|
|
|
|
# Test pgenerate
|
|
n = 10
|
|
as = [rand(4,4) for i in 1:n]
|
|
bs = deepcopy(as)
|
|
cs = collect(Base.Distributed.pgenerate(x->(sleep(rand()*0.1); svdfact(x)), bs))
|
|
svdas = map(svdfact, as)
|
|
for i in 1:n
|
|
@test cs[i][:U] ≈ svdas[i][:U]
|
|
@test cs[i][:S] ≈ svdas[i][:S]
|
|
@test cs[i][:V] ≈ svdas[i][:V]
|
|
end
|
|
|
|
# Test asyncmap
|
|
@test allunique(asyncmap(x->(sleep(1.0);object_id(current_task())), 1:10))
|
|
|
|
# num tasks
|
|
@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:20; ntasks=5))) == 5
|
|
|
|
# default num tasks
|
|
@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:200))) == 100
|
|
|
|
# ntasks as a function
|
|
let nt=0
|
|
global nt_func
|
|
nt_func() = (v=div(nt, 25); nt+=1; v) # increment number of tasks by 1 for every 25th call.
|
|
# nt_func() will be called initally once and then for every
|
|
# iteration
|
|
end
|
|
@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:200; ntasks=nt_func))) == 7
|
|
|
|
# batch mode tests
|
|
let ctr=0
|
|
global next_ctr
|
|
next_ctr() = (ctr+=1; ctr)
|
|
end
|
|
resp = asyncmap(x->(v=next_ctr(); map(_->v, x)), 1:22; ntasks=5, batch_size=5)
|
|
@test length(resp) == 22
|
|
@test length(unique(resp)) == 5
|
|
|
|
input = rand(1:1000, 100)
|
|
@test asyncmap(x->map(args->identity(args...), x), input; ntasks=5, batch_size=5) == input
|
|
|
|
# check whether shape is retained
|
|
a=rand(2,2)
|
|
b=asyncmap(identity, a)
|
|
@test a == b
|
|
@test size(a) == size(b)
|
|
|
|
# check with an iterator that does not implement size()
|
|
c=Channel(32); foreach(i->put!(c,i), 1:10); close(c)
|
|
b=asyncmap(identity, c)
|
|
@test Int[1:10...] == b
|
|
@test size(b) == (10,)
|
|
|
|
# check with an iterator that has only implements length()
|
|
len_only_iterable = (1,2,3,4,5)
|
|
@test Base.iteratorsize(len_only_iterable) == Base.HasLength()
|
|
@test asyncmap(identity, len_only_iterable) == map(identity, len_only_iterable)
|
|
|
|
# Error conditions
|
|
@test_throws ArgumentError asyncmap(identity, 1:10; batch_size=0)
|
|
@test_throws ArgumentError asyncmap(identity, 1:10; batch_size="10")
|
|
@test_throws ArgumentError asyncmap(identity, 1:10; ntasks="10")
|
|
|
|
# asyncmap and pmap with various types. Test for equivalence with map
|
|
function testmap_equivalence(f, c...)
|
|
x1 = asyncmap(f,c...)
|
|
x2 = pmap(f,c...)
|
|
x3 = map(f,c...)
|
|
|
|
if Base.iteratorsize == Base.HasShape()
|
|
@test size(x1) == size(x3)
|
|
@test size(x2) == size(x3)
|
|
else
|
|
@test length(x1) == length(x3)
|
|
@test length(x2) == length(x3)
|
|
end
|
|
|
|
@test eltype(x1) == eltype(x3)
|
|
@test eltype(x2) == eltype(x3)
|
|
|
|
for (v1,v2) in zip(x1,x3)
|
|
@test v1==v2
|
|
end
|
|
for (v1,v2) in zip(x2,x3)
|
|
@test v1==v2
|
|
end
|
|
end
|
|
|
|
testmap_equivalence(identity, (1,2,3,4))
|
|
testmap_equivalence(x->x>0?1.0:0.0, sparse(eye(5)))
|
|
testmap_equivalence((x,y,z)->x+y+z, 1,2,3)
|
|
testmap_equivalence(x->x?false:true, BitArray(10,10))
|
|
testmap_equivalence(x->"foobar", BitArray(10,10))
|
|
testmap_equivalence((x,y,z)->string(x,y,z), BitArray(10), ones(10), "1234567890")
|
|
|
|
@test asyncmap(uppercase, "Hello World!") == map(uppercase, "Hello World!")
|
|
@test pmap(uppercase, "Hello World!") == map(uppercase, "Hello World!")
|
|
|
|
|
|
# Test that the default worker pool cycles through all workers
|
|
pmap(_->myid(), 1:nworkers()) # priming run
|
|
@test nworkers() == length(unique(pmap(_->myid(), 1:100)))
|
|
|
|
# Test same behaviour when executed on a worker
|
|
@test nworkers() == length(unique(remotecall_fetch(()->pmap(_->myid(), 1:100), id_other)))
|
|
|
|
# Same tests with custom worker pools.
|
|
wp = WorkerPool(workers())
|
|
@test nworkers() == length(unique(pmap(wp, _->myid(), 1:100)))
|
|
@test nworkers() == length(unique(remotecall_fetch(wp->pmap(wp, _->myid(), 1:100), id_other, wp)))
|
|
|
|
|
|
# CachingPool tests
|
|
wp = CachingPool(workers())
|
|
@test [1:100...] == pmap(wp, x->x, 1:100)
|
|
|
|
clear!(wp)
|
|
@test length(wp.map_obj2ref) == 0
|
|
|
|
# The below block of tests are usually run only on local development systems, since:
|
|
# - tests which print errors
|
|
# - addprocs tests are memory intensive
|
|
# - ssh addprocs requires sshd to be running locally with passwordless login enabled.
|
|
# The test block is enabled by defining env JULIA_TESTFULL=1
|
|
|
|
DoFullTest = Bool(parse(Int,(get(ENV, "JULIA_TESTFULL", "0"))))
|
|
|
|
if DoFullTest
|
|
println("Testing exception printing on remote worker from a `remote_do` call")
|
|
println("Please ensure the remote error and backtrace is displayed on screen")
|
|
|
|
remote_do(id_other) do
|
|
throw(ErrorException("TESTING EXCEPTION ON REMOTE DO. PLEASE IGNORE"))
|
|
end
|
|
sleep(0.5) # Give some time for the above error to be printed
|
|
|
|
println("\n\nThe following 'invalid connection credentials' error messages are to be ignored.")
|
|
all_w = workers()
|
|
# Test sending fake data to workers. The worker processes will print an
|
|
# error message but should not terminate.
|
|
for w in Base.Distributed.PGRP.workers
|
|
if isa(w, Base.Distributed.Worker)
|
|
s = connect(get(w.config.host), get(w.config.port))
|
|
write(s, randstring(32))
|
|
end
|
|
end
|
|
@test workers() == all_w
|
|
@test all([p == remotecall_fetch(myid, p) for p in all_w])
|
|
|
|
if is_unix() # aka have ssh
|
|
function test_n_remove_pids(new_pids)
|
|
for p in new_pids
|
|
w_in_remote = sort(remotecall_fetch(workers, p))
|
|
try
|
|
@test intersect(new_pids, w_in_remote) == new_pids
|
|
catch e
|
|
print("p : $p\n")
|
|
print("newpids : $new_pids\n")
|
|
print("w_in_remote : $w_in_remote\n")
|
|
print("intersect : $(intersect(new_pids, w_in_remote))\n\n\n")
|
|
rethrow(e)
|
|
end
|
|
end
|
|
|
|
remotecall_fetch(rmprocs, 1, new_pids)
|
|
end
|
|
|
|
print("\n\nTesting SSHManager. A minimum of 4GB of RAM is recommended.\n")
|
|
print("Please ensure: \n")
|
|
print("1) sshd is running locally with passwordless login enabled.\n")
|
|
print("2) Env variable USER is defined and is the ssh user.\n")
|
|
print("3) Port 9300 is not in use.\n")
|
|
|
|
sshflags = `-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR `
|
|
#Issue #9951
|
|
hosts=[]
|
|
localhost_aliases = ["localhost", string(getipaddr()), "127.0.0.1"]
|
|
num_workers = parse(Int,(get(ENV, "JULIA_ADDPROCS_NUM", "9")))
|
|
|
|
for i in 1:(num_workers/length(localhost_aliases))
|
|
append!(hosts, localhost_aliases)
|
|
end
|
|
|
|
print("\nTesting SSH addprocs with $(length(hosts)) workers...\n")
|
|
new_pids = addprocs_with_testenv(hosts; sshflags=sshflags)
|
|
@test length(new_pids) == length(hosts)
|
|
test_n_remove_pids(new_pids)
|
|
|
|
print("\nMixed ssh addprocs with :auto\n")
|
|
new_pids = addprocs_with_testenv(["localhost", ("127.0.0.1", :auto), "localhost"]; sshflags=sshflags)
|
|
@test length(new_pids) == (2 + Sys.CPU_CORES)
|
|
test_n_remove_pids(new_pids)
|
|
|
|
print("\nMixed ssh addprocs with numeric counts\n")
|
|
new_pids = addprocs_with_testenv([("localhost", 2), ("127.0.0.1", 2), "localhost"]; sshflags=sshflags)
|
|
@test length(new_pids) == 5
|
|
test_n_remove_pids(new_pids)
|
|
|
|
print("\nssh addprocs with tunnel\n")
|
|
new_pids = addprocs_with_testenv([("localhost", num_workers)]; tunnel=true, sshflags=sshflags)
|
|
@test length(new_pids) == num_workers
|
|
test_n_remove_pids(new_pids)
|
|
|
|
print("\nAll supported formats for hostname\n")
|
|
h1 = "localhost"
|
|
user = ENV["USER"]
|
|
h2 = "$user@$h1"
|
|
h3 = "$h2:22"
|
|
h4 = "$h3 $(string(getipaddr()))"
|
|
h5 = "$h4:9300"
|
|
|
|
new_pids = addprocs_with_testenv([h1, h2, h3, h4, h5]; sshflags=sshflags)
|
|
@test length(new_pids) == 5
|
|
test_n_remove_pids(new_pids)
|
|
|
|
print("\nkeyword arg exename\n")
|
|
for exename in [`$(joinpath(JULIA_HOME, Base.julia_exename()))`, "$(joinpath(JULIA_HOME, Base.julia_exename()))"]
|
|
for addp_func in [()->addprocs_with_testenv(["localhost"]; exename=exename, exeflags=test_exeflags, sshflags=sshflags),
|
|
()->addprocs_with_testenv(1; exename=exename, exeflags=test_exeflags)]
|
|
|
|
new_pids = addp_func()
|
|
@test length(new_pids) == 1
|
|
test_n_remove_pids(new_pids)
|
|
end
|
|
end
|
|
|
|
end # unix-only
|
|
end # full-test
|
|
|
|
let t = @task 42
|
|
schedule(t, ErrorException(""), error=true)
|
|
@test_throws ErrorException wait(t)
|
|
end
|
|
|
|
# issue #8207
|
|
let A = Any[]
|
|
@parallel (+) for i in (push!(A,1); 1:2)
|
|
i
|
|
end
|
|
@test length(A) == 1
|
|
end
|
|
|
|
# issue #13168
|
|
function f13168(n)
|
|
val = 0
|
|
for i=1:n val+=sum(rand(n,n)^2) end
|
|
val
|
|
end
|
|
let t = schedule(@task f13168(100))
|
|
@test t.state == :queued
|
|
@test_throws ErrorException schedule(t)
|
|
yield()
|
|
@test t.state == :done
|
|
@test_throws ErrorException schedule(t)
|
|
@test isa(wait(t),Float64)
|
|
end
|
|
|
|
# issue #13122
|
|
@test remotecall_fetch(identity, workers()[1], C_NULL) === C_NULL
|
|
|
|
# issue #11062
|
|
function t11062()
|
|
@async v11062 = 1
|
|
v11062 = 2
|
|
end
|
|
|
|
@test t11062() == 2
|
|
|
|
# issue #15406
|
|
v15406 = remotecall_wait(() -> 1, id_other)
|
|
fetch(v15406)
|
|
remotecall_wait(fetch, id_other, v15406)
|
|
|
|
# Test various forms of remotecall* invocations
|
|
|
|
@everywhere f_args(v1, v2=0; kw1=0, kw2=0) = v1+v2+kw1+kw2
|
|
|
|
function test_f_args(result, args...; kwargs...)
|
|
@test fetch(remotecall(args...; kwargs...)) == result
|
|
@test fetch(remotecall_wait(args...; kwargs...)) == result
|
|
@test remotecall_fetch(args...; kwargs...) == result
|
|
|
|
# A visual test - remote_do should NOT print any errors
|
|
remote_do(args...; kwargs...)
|
|
end
|
|
|
|
for tid in [id_other, id_me, Base.default_worker_pool()]
|
|
test_f_args(1, f_args, tid, 1)
|
|
test_f_args(3, f_args, tid, 1, 2)
|
|
test_f_args(5, f_args, tid, 1; kw1=4)
|
|
test_f_args(13, f_args, tid, 1; kw1=4, kw2=8)
|
|
test_f_args(15, f_args, tid, 1, 2; kw1=4, kw2=8)
|
|
end
|
|
|
|
# Test remote_do
|
|
f=Future(id_me)
|
|
remote_do(fut->put!(fut, myid()), id_me, f)
|
|
@test fetch(f) == id_me
|
|
|
|
f=Future(id_other)
|
|
remote_do(fut->put!(fut, myid()), id_other, f)
|
|
@test fetch(f) == id_other
|
|
|
|
# github PR #14456
|
|
n = DoFullTest ? 6 : 5
|
|
for i = 1:10^n
|
|
fetch(@spawnat myid() myid())
|
|
end
|
|
|
|
# issue #15451
|
|
@test remotecall_fetch(x->(y->2y)(x)+1, workers()[1], 3) == 7
|
|
|
|
# issue #16091
|
|
mutable struct T16091 end
|
|
wid = workers()[1]
|
|
@test try
|
|
remotecall_fetch(()->T16091, wid)
|
|
false
|
|
catch ex
|
|
((ex::RemoteException).captured::CapturedException).ex === UndefVarError(:T16091)
|
|
end
|
|
@test try
|
|
remotecall_fetch(identity, wid, T16091)
|
|
false
|
|
catch ex
|
|
((ex::RemoteException).captured::CapturedException).ex === UndefVarError(:T16091)
|
|
end
|
|
|
|
f16091a() = 1
|
|
remotecall_fetch(()->eval(:(f16091a() = 2)), wid)
|
|
@test remotecall_fetch(f16091a, wid) === 2
|
|
@test remotecall_fetch((myid)->remotecall_fetch(f16091a, myid), wid, myid()) === 1
|
|
|
|
# these will only heisen-fail, since it depends on the gensym counter collisions:
|
|
f16091b = () -> 1
|
|
remotecall_fetch(()->eval(:(f16091b = () -> 2)), wid)
|
|
@test remotecall_fetch(f16091b, 2) === 1
|
|
# Global anonymous functions are over-written...
|
|
@test remotecall_fetch((myid)->remotecall_fetch(f16091b, myid), wid, myid()) === 1
|
|
|
|
# ...while local anonymous functions are by definition, local.
|
|
let
|
|
f16091c = () -> 1
|
|
@test remotecall_fetch(f16091c, 2) === 1
|
|
@test remotecall_fetch(
|
|
myid -> begin
|
|
let
|
|
f16091c = () -> 2
|
|
remotecall_fetch(f16091c, myid)
|
|
end
|
|
end, wid, myid()) === 2
|
|
end
|
|
|
|
# issue #16451
|
|
rng=RandomDevice()
|
|
retval = @parallel (+) for _ in 1:10
|
|
rand(rng)
|
|
end
|
|
@test retval > 0.0 && retval < 10.0
|
|
|
|
rand(rng)
|
|
retval = @parallel (+) for _ in 1:10
|
|
rand(rng)
|
|
end
|
|
@test retval > 0.0 && retval < 10.0
|
|
|
|
# serialization tests
|
|
wrkr1 = workers()[1]
|
|
wrkr2 = workers()[end]
|
|
|
|
@test remotecall_fetch(p->remotecall_fetch(myid, p), wrkr1, wrkr2) == wrkr2
|
|
|
|
# Send f to wrkr1 and wrkr2. Then try calling f on wrkr2 from wrkr1
|
|
f_myid = ()->myid()
|
|
@test wrkr1 == remotecall_fetch(f_myid, wrkr1)
|
|
@test wrkr2 == remotecall_fetch(f_myid, wrkr2)
|
|
@test wrkr2 == remotecall_fetch((f, p)->remotecall_fetch(f, p), wrkr1, f_myid, wrkr2)
|
|
|
|
# Deserialization error recovery test
|
|
# locally defined module, but unavailable on workers
|
|
module LocalFoo
|
|
global foo=1
|
|
end
|
|
|
|
let
|
|
@test_throws RemoteException remotecall_fetch(()->LocalFoo.foo, 2)
|
|
|
|
bad_thunk = ()->NonexistantModule.f()
|
|
@test_throws RemoteException remotecall_fetch(bad_thunk, 2)
|
|
|
|
# Test that the stream is still usable
|
|
@test remotecall_fetch(()->:test,2) == :test
|
|
ref = remotecall(bad_thunk, 2)
|
|
@test_throws RemoteException fetch(ref)
|
|
end
|
|
|
|
# Test calling @everywhere from a module not defined on the workers
|
|
module LocalBar
|
|
bar() = @everywhere new_bar()=myid()
|
|
end
|
|
LocalBar.bar()
|
|
for p in procs()
|
|
@test p == remotecall_fetch(new_bar, p)
|
|
end
|
|
|
|
# Test addprocs enable_threaded_blas parameter
|
|
|
|
const get_num_threads = function() # anonymous so it will be serialized when called
|
|
blas = BLAS.vendor()
|
|
# Wrap in a try to catch unsupported blas versions
|
|
try
|
|
if blas == :openblas
|
|
return ccall((:openblas_get_num_threads, Base.libblas_name), Cint, ())
|
|
elseif blas == :openblas64
|
|
return ccall((:openblas_get_num_threads64_, Base.libblas_name), Cint, ())
|
|
elseif blas == :mkl
|
|
return ccall((:MKL_Get_Max_Num_Threads, Base.libblas_name), Cint, ())
|
|
end
|
|
|
|
# OSX BLAS looks at an environment variable
|
|
if is_apple()
|
|
return ENV["VECLIB_MAXIMUM_THREADS"]
|
|
end
|
|
end
|
|
|
|
return nothing
|
|
end
|
|
|
|
function get_remote_num_threads(processes_added)
|
|
return [remotecall_fetch(get_num_threads, proc_id) for proc_id in processes_added]
|
|
end
|
|
|
|
function test_blas_config(pid, expected)
|
|
for worker in Base.Distributed.PGRP.workers
|
|
if worker.id == pid
|
|
@test get(worker.config.enable_threaded_blas) == expected
|
|
return
|
|
end
|
|
end
|
|
end
|
|
|
|
function test_add_procs_threaded_blas()
|
|
if get_num_threads() === nothing
|
|
warn("Skipping blas num threads tests due to unsupported blas version")
|
|
return
|
|
end
|
|
master_blas_thread_count = get_num_threads()
|
|
|
|
# Test with default enable_threaded_blas false
|
|
processes_added = addprocs_with_testenv(2)
|
|
for proc_id in processes_added
|
|
test_blas_config(proc_id, false)
|
|
end
|
|
|
|
# Master thread should not have changed
|
|
@test get_num_threads() == master_blas_thread_count
|
|
|
|
# Threading disabled in children by default
|
|
thread_counts_by_process = get_remote_num_threads(processes_added)
|
|
for thread_count in thread_counts_by_process
|
|
@test thread_count == 1
|
|
end
|
|
rmprocs(processes_added)
|
|
|
|
processes_added = addprocs_with_testenv(2, enable_threaded_blas=true)
|
|
for proc_id in processes_added
|
|
test_blas_config(proc_id, true)
|
|
end
|
|
|
|
@test get_num_threads() == master_blas_thread_count
|
|
|
|
# BLAS.set_num_threads(`num`) doesn't cause get_num_threads to return `num`
|
|
# depending on the machine, the BLAS version, and BLAS configuration, so
|
|
# we need a very lenient test.
|
|
thread_counts_by_process = get_remote_num_threads(processes_added)
|
|
for thread_count in thread_counts_by_process
|
|
@test thread_count >= 1
|
|
end
|
|
rmprocs(processes_added)
|
|
end
|
|
test_add_procs_threaded_blas()
|
|
|
|
#19687
|
|
# ensure no race conditions between rmprocs and addprocs
|
|
for i in 1:5
|
|
p = addprocs_with_testenv(1)[1]
|
|
@spawnat p sleep(5)
|
|
rmprocs(p; waitfor=0)
|
|
end
|
|
|
|
# Test if a wait has been called on rmprocs(...;waitfor=0), further remotecalls
|
|
# don't throw errors.
|
|
for i in 1:5
|
|
p = addprocs_with_testenv(1)[1]
|
|
np = nprocs()
|
|
@spawnat p sleep(5)
|
|
wait(rmprocs(p; waitfor=0))
|
|
for pid in procs()
|
|
@test pid == remotecall_fetch(myid, pid)
|
|
end
|
|
@test nprocs() == np - 1
|
|
end
|
|
|
|
# Test that an exception is thrown if workers are unable to be removed within requested time.
|
|
if DoFullTest
|
|
pids=addprocs_with_testenv(4);
|
|
@test_throws ErrorException rmprocs(pids; waitfor=0.001);
|
|
# wait for workers to be removed
|
|
while any(x -> (x in procs()), pids)
|
|
sleep(0.1)
|
|
end
|
|
end
|
|
|
|
# Test addprocs/rmprocs from master node only
|
|
for f in [ ()->addprocs(1; exeflags=test_exeflags), ()->rmprocs(workers()) ]
|
|
try
|
|
remotecall_fetch(f, id_other)
|
|
error("Unexpected")
|
|
catch ex
|
|
@test isa(ex, RemoteException)
|
|
@test ex.captured.ex.msg == "Only process 1 can add and remove workers"
|
|
end
|
|
end
|
|
|
|
# Test the following addprocs error conditions
|
|
# - invalid host name - github issue #20372
|
|
# - julia exe exiting with an error
|
|
# - timeout reading host:port from worker STDOUT
|
|
# - host:port not found in worker STDOUT in the first 1000 lines
|
|
|
|
struct ErrorSimulator <: ClusterManager
|
|
mode
|
|
end
|
|
|
|
function Base.launch(manager::ErrorSimulator, params::Dict, launched::Array, c::Condition)
|
|
exename = params[:exename]
|
|
dir = params[:dir]
|
|
|
|
cmd = `$(Base.julia_cmd(exename)) --startup-file=no`
|
|
if manager.mode == :timeout
|
|
cmd = `$cmd -e "sleep(10)"`
|
|
elseif manager.mode == :ntries
|
|
cmd = `$cmd -e "[println(x) for x in 1:1001]"`
|
|
elseif manager.mode == :exit
|
|
cmd = `$cmd -e "exit(-1)"`
|
|
else
|
|
error("Unknown mode")
|
|
end
|
|
io, pobj = open(pipeline(detach(setenv(cmd, dir=dir)); stderr=STDERR), "r")
|
|
|
|
wconfig = WorkerConfig()
|
|
wconfig.process = pobj
|
|
wconfig.io = io
|
|
push!(launched, wconfig)
|
|
notify(c)
|
|
end
|
|
|
|
testruns = Any[]
|
|
|
|
if DoFullTest
|
|
append!(testruns, [(()->addprocs_with_testenv(["errorhost20372"]), "Unable to read host:port string from worker. Launch command exited with error?", ())])
|
|
end
|
|
|
|
append!(testruns, [
|
|
(()->addprocs_with_testenv(ErrorSimulator(:exit)), "Unable to read host:port string from worker. Launch command exited with error?", ()),
|
|
(()->addprocs_with_testenv(ErrorSimulator(:ntries)), "Unexpected output from worker launch command. Host:port string not found.", ()),
|
|
(()->addprocs_with_testenv(ErrorSimulator(:timeout)), "Timed out waiting to read host:port string from worker.", ("JULIA_WORKER_TIMEOUT"=>"1",))
|
|
])
|
|
|
|
for (addp_testf, expected_errstr, env) in testruns
|
|
try
|
|
withenv(env...) do
|
|
addp_testf()
|
|
end
|
|
error("Unexpected")
|
|
catch ex
|
|
@test isa(ex, CompositeException)
|
|
@test ex.exceptions[1].ex.msg == expected_errstr
|
|
end
|
|
end
|
|
|
|
|
|
# Auto serialization of globals from Main.
|
|
# bitstypes
|
|
global v1 = 1
|
|
@test remotecall_fetch(()->v1, id_other) == v1
|
|
@test remotecall_fetch(()->isdefined(Main, :v1), id_other)
|
|
for i in 2:5
|
|
global v1 = i
|
|
@test remotecall_fetch(()->v1, id_other) == i
|
|
end
|
|
|
|
# non-bitstypes
|
|
global v2 = zeros(10)
|
|
for i in 1:5
|
|
v2[i] = i
|
|
@test remotecall_fetch(()->v2, id_other) == v2
|
|
end
|
|
|
|
# Different global bindings to the same object
|
|
global v3 = ones(10)
|
|
global v4 = v3
|
|
@test remotecall_fetch(()->v3, id_other) == remotecall_fetch(()->v4, id_other)
|
|
@test remotecall_fetch(()->isdefined(Main, :v3), id_other)
|
|
@test remotecall_fetch(()->isdefined(Main, :v4), id_other)
|
|
|
|
# Global references to Types and Modules should work if they are locally defined
|
|
global v5 = Int
|
|
global v6 = Base.Distributed
|
|
@test remotecall_fetch(()->v5, id_other) === Int
|
|
@test remotecall_fetch(()->v6, id_other) === Base.Distributed
|
|
|
|
struct FooStructLocal end
|
|
module FooModLocal end
|
|
v5 = FooStructLocal
|
|
v6 = FooModLocal
|
|
@test_throws RemoteException remotecall_fetch(()->v5, id_other)
|
|
@test_throws RemoteException remotecall_fetch(()->v6, id_other)
|
|
|
|
@everywhere struct FooStructEverywhere end
|
|
@everywhere module FooModEverywhere end
|
|
v5 = FooStructEverywhere
|
|
v6 = FooModEverywhere
|
|
@test remotecall_fetch(()->v5, id_other) === FooStructEverywhere
|
|
@test remotecall_fetch(()->v6, id_other) === FooModEverywhere
|
|
|
|
|
|
# Test that a global is not being repeatedly serialized when
|
|
# a) referenced multiple times in the closure
|
|
# b) hash value has not changed.
|
|
|
|
@everywhere begin
|
|
global testsercnt_d = Dict()
|
|
mutable struct TestSerCnt
|
|
v
|
|
end
|
|
import Base.hash, Base.==
|
|
hash(x::TestSerCnt, h::UInt) = hash(hash(x.v), h)
|
|
==(x1::TestSerCnt, x2::TestSerCnt) = (x1.v == x2.v)
|
|
|
|
function Base.serialize(s::AbstractSerializer, t::TestSerCnt)
|
|
Base.Serializer.serialize_type(s, TestSerCnt)
|
|
serialize(s, t.v)
|
|
global testsercnt_d
|
|
cnt = get!(testsercnt_d, object_id(t), 0)
|
|
testsercnt_d[object_id(t)] = cnt+1
|
|
end
|
|
|
|
Base.deserialize(s::AbstractSerializer, ::Type{TestSerCnt}) = TestSerCnt(deserialize(s))
|
|
end
|
|
|
|
# hash value of tsc is not changed
|
|
global tsc = TestSerCnt(zeros(10))
|
|
for i in 1:5
|
|
remotecall_fetch(()->tsc, id_other)
|
|
end
|
|
# should have been serialized only once
|
|
@test testsercnt_d[object_id(tsc)] == 1
|
|
|
|
# hash values are changed
|
|
n=5
|
|
testsercnt_d[object_id(tsc)] = 0
|
|
for i in 1:n
|
|
tsc.v[i] = i
|
|
remotecall_fetch(()->tsc, id_other)
|
|
end
|
|
# should have been serialized as many times as the loop
|
|
@test testsercnt_d[object_id(tsc)] == n
|
|
|
|
# Multiple references in a closure should be serialized only once.
|
|
global mrefs = TestSerCnt(ones(10))
|
|
@test remotecall_fetch(()->(mrefs.v, 2*mrefs.v, 3*mrefs.v), id_other) == (ones(10), 2*ones(10), 3*ones(10))
|
|
@test testsercnt_d[object_id(mrefs)] == 1
|
|
|
|
|
|
# nested anon functions
|
|
global f1 = x->x
|
|
global f2 = x->f1(x)
|
|
v = rand()
|
|
@test remotecall_fetch(f2, id_other, v) == v
|
|
@test remotecall_fetch(x->f2(x), id_other, v) == v
|
|
|
|
# consts
|
|
const c1 = ones(10)
|
|
@test remotecall_fetch(()->c1, id_other) == c1
|
|
@test remotecall_fetch(()->isconst(Main, :c1), id_other)
|
|
|
|
# Test same calls with local vars
|
|
function wrapped_var_ser_tests()
|
|
# bitstypes
|
|
local lv1 = 1
|
|
@test remotecall_fetch(()->lv1, id_other) == lv1
|
|
@test !remotecall_fetch(()->isdefined(Main, :lv1), id_other)
|
|
for i in 2:5
|
|
lv1 = i
|
|
@test remotecall_fetch(()->lv1, id_other) == i
|
|
end
|
|
|
|
# non-bitstypes
|
|
local lv2 = zeros(10)
|
|
for i in 1:5
|
|
lv2[i] = i
|
|
@test remotecall_fetch(()->lv2, id_other) == lv2
|
|
end
|
|
|
|
# nested anon functions
|
|
local lf1 = x->x
|
|
local lf2 = x->lf1(x)
|
|
v = rand()
|
|
@test remotecall_fetch(lf2, id_other, v) == v
|
|
@test remotecall_fetch(x->lf2(x), id_other, v) == v
|
|
end
|
|
|
|
wrapped_var_ser_tests()
|
|
|
|
# Test internal data structures being cleaned up upon gc.
|
|
global ids_cleanup = ones(6)
|
|
global ids_func = ()->ids_cleanup
|
|
|
|
clust_ser = (Base.Distributed.worker_from_id(id_other)).w_serializer
|
|
@test remotecall_fetch(ids_func, id_other) == ids_cleanup
|
|
|
|
@test haskey(clust_ser.glbs_sent, object_id(ids_cleanup))
|
|
finalize(ids_cleanup)
|
|
@test !haskey(clust_ser.glbs_sent, object_id(ids_cleanup))
|
|
|
|
# TODO Add test for cleanup from `clust_ser.glbs_in_tnobj`
|
|
|
|
# reported github issues - Mostly tests with globals and various distributed macros
|
|
#2669, #5390
|
|
v2669=10
|
|
@test fetch(@spawn (1+v2669)) == 11
|
|
|
|
#12367
|
|
refs = []
|
|
if true
|
|
n = 10
|
|
for p in procs()
|
|
push!(refs, @spawnat p begin
|
|
@sync for i in 1:n
|
|
nothing
|
|
end
|
|
end)
|
|
end
|
|
end
|
|
foreach(wait, refs)
|
|
|
|
#14399
|
|
s = convert(SharedArray, [1,2,3,4])
|
|
@test pmap(i->length(s), 1:2) == [4,4]
|
|
|
|
#6760
|
|
if true
|
|
a = 2
|
|
x = @parallel (vcat) for k=1:2
|
|
sin(a)
|
|
end
|
|
end
|
|
@test x == map(_->sin(2), 1:2)
|
|
|
|
let thrown = false
|
|
try
|
|
remotecall_fetch(sqrt, 2, -1)
|
|
catch e
|
|
thrown = true
|
|
b = IOBuffer()
|
|
showerror(b, e)
|
|
@test contains(String(take!(b)), "sqrt will only return")
|
|
end
|
|
@test thrown
|
|
end
|
|
|
|
#19463
|
|
function foo19463()
|
|
w1 = workers()[1]
|
|
w2 = workers()[2]
|
|
w3 = workers()[3]
|
|
|
|
b1 = () -> 1
|
|
b2 = () -> fetch(@spawnat w1 b1()) + 1
|
|
b3 = () -> fetch(@spawnat w2 b2()) + 1
|
|
b4 = () -> fetch(@spawnat w3 b3()) + 1
|
|
b4()
|
|
end
|
|
@test foo19463() == 4
|
|
|
|
# Testing clear!
|
|
function setup_syms(n, pids)
|
|
syms = []
|
|
for i in 1:n
|
|
symstr = string("clrtest", randstring())
|
|
sym = Symbol(symstr)
|
|
eval(:(global $sym = rand()))
|
|
for p in pids
|
|
eval(:(@test $sym == remotecall_fetch(()->$sym, $p)))
|
|
eval(:(@test remotecall_fetch(isdefined, $p, Symbol($symstr))))
|
|
end
|
|
push!(syms, sym)
|
|
end
|
|
syms
|
|
end
|
|
|
|
function test_clear(syms, pids)
|
|
for p in pids
|
|
for sym in syms
|
|
remote_val = remotecall_fetch(()->getfield(Main, sym), p)
|
|
@test remote_val === nothing
|
|
@test remote_val != getfield(Main, sym)
|
|
end
|
|
end
|
|
end
|
|
|
|
syms = setup_syms(1, [id_other])
|
|
clear!(syms[1], id_other)
|
|
test_clear(syms, [id_other])
|
|
|
|
syms = setup_syms(1, workers())
|
|
clear!(syms[1], workers())
|
|
test_clear(syms, workers())
|
|
|
|
syms = setup_syms(3, [id_other])
|
|
clear!(syms, id_other)
|
|
test_clear(syms, [id_other])
|
|
|
|
syms = setup_syms(3, workers())
|
|
clear!(syms, workers())
|
|
test_clear(syms, workers())
|
|
|
|
# Test partial recovery from a deserialization error in CapturedException
|
|
try
|
|
expr = quote
|
|
mutable struct DontExistOn1
|
|
x
|
|
end
|
|
throw(BoundsError(DontExistOn1(1), 1))
|
|
end
|
|
|
|
remotecall_fetch(()->eval(expr), id_other)
|
|
error("unexpected")
|
|
catch ex
|
|
@test isa(ex.captured.ex.exceptions[1].ex, ErrorException)
|
|
@test contains(ex.captured.ex.exceptions[1].ex.msg, "BoundsError")
|
|
@test ex.captured.ex.exceptions[2].ex == UndefVarError(:DontExistOn1)
|
|
end
|
|
|
|
@test let
|
|
# creates a new worker in the same folder and tries to include file
|
|
tmp_file, temp_file_stream = mktemp()
|
|
close(temp_file_stream)
|
|
tmp_file = relpath(tmp_file)
|
|
try
|
|
proc = addprocs_with_testenv(1)
|
|
include(tmp_file)
|
|
remotecall_fetch(include, proc[1], tmp_file)
|
|
rmprocs(proc)
|
|
rm(tmp_file)
|
|
return true
|
|
catch e
|
|
println(e)
|
|
rm(tmp_file, force=true)
|
|
return false
|
|
end
|
|
end == true
|
|
|
|
@test let
|
|
# creates a new worker in the different folder and tries to include file
|
|
tmp_file, temp_file_stream = mktemp()
|
|
close(temp_file_stream)
|
|
tmp_file = relpath(tmp_file)
|
|
tmp_dir = relpath(mktempdir())
|
|
try
|
|
proc = addprocs_with_testenv(1, dir=tmp_dir)
|
|
include(tmp_file)
|
|
remotecall_fetch(include, proc[1], tmp_file)
|
|
rmprocs(proc)
|
|
rm(tmp_dir)
|
|
rm(tmp_file)
|
|
return true
|
|
catch e
|
|
println(e)
|
|
rm(tmp_dir, force=true)
|
|
rm(tmp_file, force=true)
|
|
return false
|
|
end
|
|
end == true
|
|
|
|
# Run topology tests last after removing all workers, since a given
|
|
# cluster at any time only supports a single topology.
|
|
rmprocs(workers())
|
|
include("topology.jl")
|