Add: julia-0.6.2

Former-commit-id: ccc667cf67d569f3fb3df39aa57c2134755a7551
This commit is contained in:
2018-02-10 10:27:19 -07:00
parent 94220957d7
commit 019f8e3064
723 changed files with 276164 additions and 0 deletions

View File

@@ -0,0 +1,27 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
module ModInts
export ModInt
import Base: +, -, *, /, inv
struct ModInt{n} <: Integer
k::Int
ModInt{n}(k) where n = new(mod(k,n))
end
Base.show{n}(io::IO, k::ModInt{n}) =
print(io, get(io, :compact, false) ? k.k : "$(k.k) mod $n")
+{n}(a::ModInt{n}, b::ModInt{n}) = ModInt{n}(a.k+b.k)
-{n}(a::ModInt{n}, b::ModInt{n}) = ModInt{n}(a.k-b.k)
*{n}(a::ModInt{n}, b::ModInt{n}) = ModInt{n}(a.k*b.k)
-{n}(a::ModInt{n}) = ModInt{n}(-a.k)
inv{n}(a::ModInt{n}) = ModInt{n}(invmod(a.k, n))
/{n}(a::ModInt{n}, b::ModInt{n}) = a*inv(b) # broaden for non-coprime?
Base.promote_rule{n}(::Type{ModInt{n}}, ::Type{Int}) = ModInt{n}
Base.convert{n}(::Type{ModInt{n}}, i::Int) = ModInt{n}(i)
end # module

View File

@@ -0,0 +1,19 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
import Base.Sort
struct BubbleSortAlg <: Sort.Algorithm end
const BubbleSort = BubbleSortAlg()
function Base.sort!(v::AbstractVector, lo::Int, hi::Int, ::BubbleSortAlg, o::Sort.Ordering)
while true
clean = true
for i = lo:hi-1
if Sort.lt(o, v[i+1], v[i])
v[i+1], v[i] = v[i], v[i+1]
clean = false
end
end
clean && break
end
return v
end

View File

@@ -0,0 +1,27 @@
This is a proof-of-concept that uses ZeroMQ as transport.
It uses a star topology as opposed to the native mesh network.
Package ZMQ must be installed. All workers only run on localhost.
All Julia nodes only connect to a "broker" process that listens on known ports
8100 and 8101 via ZMQ sockets.
All commands must be run from `examples/clustermanager/0mq` directory
First, start the broker. In a new console type:
julia broker.jl
This does not return.
Next, start a Julia REPL and type:
include("ZMQCM.jl")
ZMQCM.start_master(4) # start with four workers
Alternatively, head.jl, a test script could be run. It just launches the requested number of workers,
executes a simple command on all of them and exits.
julia head.jl 4
NOTE: As stated this is a proof-of-concept. A real Julia cluster using ZMQ will probably use
different ZMQ socket types and optimize the transport.

View File

@@ -0,0 +1,277 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
using ZMQ
import Base: launch, manage, connect, kill
const BROKER_SUB_PORT = 8100
const BROKER_PUB_PORT = 8101
const SELF_INITIATED = 0
const REMOTE_INITIATED = 1
const PAYLOAD_MSG = "J"
const CONTROL_MSG = "Z"
const REQUEST_ACK = "R"
const ACK_MSG = "A"
const KILL_MSG = "K"
mutable struct ZMQCMan <: ClusterManager
map_zmq_julia::Dict{Int, Tuple}
c::Condition
isfree::Bool
ctx
pub
sub
zid_self
ZMQCMan() = new(Dict{Int, Tuple}(), Condition(), true)
end
const manager = ZMQCMan()
function lock_for_send()
if manager.isfree == true
manager.isfree = false
else
while manager.isfree == false
wait(manager.c)
if manager.isfree == true
manager.isfree = false
return
end
end
end
end
function release_lock_for_send()
manager.isfree = true
notify(manager.c, all=true)
end
function init_node(zid=0)
manager.ctx = Context(1)
pub=Socket(manager.ctx, PUB) # Outbound
connect(pub, "tcp://127.0.0.1:$BROKER_SUB_PORT")
sub=Socket(manager.ctx, SUB) # In bound
connect(sub, "tcp://127.0.0.1:$BROKER_PUB_PORT")
ZMQ.set_subscribe(sub, string(zid))
manager.pub = pub
manager.sub = sub
manager.zid_self = zid
(pub, sub)
end
function send_data(zid, mtype, data)
lock_for_send()
ZMQ.send(manager.pub, Message(string(zid)), SNDMORE)
ZMQ.send(manager.pub, Message(string(manager.zid_self)), SNDMORE)
#println("Sending message of type $mtype to $zid")
ZMQ.send(manager.pub, Message(mtype), SNDMORE)
ZMQ.send(manager.pub, Message(data))
release_lock_for_send()
end
function setup_connection(zid, initiated_by)
try
read_stream=BufferStream()
write_stream=BufferStream()
if initiated_by == REMOTE_INITIATED
test_remote = false
else
test_remote = true
end
manager.map_zmq_julia[zid] = (read_stream, write_stream, test_remote)
@schedule begin
while true
(r_s, w_s, do_test_remote) = manager.map_zmq_julia[zid]
if do_test_remote
send_data(zid, CONTROL_MSG, REQUEST_ACK)
sleep(0.5)
else
break
end
end
(r_s, w_s, do_test_remote) = manager.map_zmq_julia[zid]
while true
data = readavailable(w_s)
send_data(zid, PAYLOAD_MSG, data)
end
end
(read_stream, write_stream)
catch e
Base.show_backtrace(STDOUT,catch_backtrace())
println(e)
rethrow(e)
end
end
# BROKER
function start_broker()
ctx=Context(1)
xpub=Socket(ctx, XPUB)
xsub=Socket(ctx, XSUB)
ZMQ.bind(xsub, "tcp://127.0.0.1:$(BROKER_SUB_PORT)")
ZMQ.bind(xpub, "tcp://127.0.0.1:$(BROKER_PUB_PORT)")
ccall((:zmq_proxy, :libzmq), Cint, (Ptr{Void}, Ptr{Void}, Ptr{Void}), xsub.data, xpub.data, C_NULL)
# proxy(xsub, xpub)
# control never comes here
ZMQ.close(xpub)
ZMQ.close(xsub)
ZMQ.close(ctx)
end
function recv_data()
try
#println("On $(manager.zid_self) waiting to recv message")
zid = parse(Int,String(ZMQ.recv(manager.sub)))
assert(zid == manager.zid_self)
from_zid = parse(Int,String(ZMQ.recv(manager.sub)))
mtype = String(ZMQ.recv(manager.sub))
#println("$zid received message of type $mtype from $from_zid")
data = ZMQ.recv(manager.sub)
if mtype == CONTROL_MSG
cmsg = String(data)
if cmsg == REQUEST_ACK
#println("$from_zid REQUESTED_ACK from $zid")
# send back a control_msg
send_data(from_zid, CONTROL_MSG, ACK_MSG)
elseif cmsg == ACK_MSG
#println("$zid got ACK_MSG from $from_zid")
(r_s, w_s, test_remote) = manager.map_zmq_julia[from_zid]
manager.map_zmq_julia[from_zid] = (r_s, w_s, false)
elseif cmsg == KILL_MSG
exit(0)
else
error("Unknown control message : ", cmsg)
end
data = ""
end
(from_zid, data)
catch e
Base.show_backtrace(STDOUT,catch_backtrace())
println(e)
rethrow(e)
end
end
# MASTER
function start_master(np)
init_node()
@schedule begin
try
while true
(from_zid, data) = recv_data()
#println("master recv data from $from_zid")
(r_s, w_s, t_r) = manager.map_zmq_julia[from_zid]
unsafe_write(r_s, pointer(data), length(data))
end
catch e
Base.show_backtrace(STDOUT,catch_backtrace())
println(e)
rethrow(e)
end
end
addprocs(manager; np=np)
end
function launch(manager::ZMQCMan, params::Dict, launched::Array, c::Condition)
#println("launch $(params[:np])")
for i in 1:params[:np]
io, pobj = open(`$(params[:exename]) worker.jl $i $(Base.cluster_cookie())`, "r")
wconfig = WorkerConfig()
wconfig.userdata = Dict(:zid=>i, :io=>io)
push!(launched, wconfig)
notify(c)
end
end
function connect(manager::ZMQCMan, pid::Int, config::WorkerConfig)
#println("connect_m2w")
if myid() == 1
zid = get(config.userdata)[:zid]
config.connect_at = zid # This will be useful in the worker-to-worker connection setup.
print_worker_stdout(get(config.userdata)[:io], pid)
else
#println("connect_w2w")
zid = get(config.connect_at)
config.userdata = Dict{Symbol, Any}(:zid=>zid)
end
streams = setup_connection(zid, SELF_INITIATED)
udata = get(config.userdata)
udata[:streams] = streams
streams
end
# WORKER
function start_worker(zid, cookie)
#println("start_worker")
Base.init_worker(cookie, ZMQCMan())
init_node(zid)
while true
(from_zid, data) = recv_data()
#println("worker recv data from $from_zid")
streams = get(manager.map_zmq_julia, from_zid, nothing)
if streams === nothing
# First time..
(r_s, w_s) = setup_connection(from_zid, REMOTE_INITIATED)
Base.process_messages(r_s, w_s)
else
(r_s, w_s, t_r) = streams
end
unsafe_write(r_s, pointer(data), length(data))
end
end
function manage(manager::ZMQCMan, id::Int, config::WorkerConfig, op)
nothing
end
function kill(manager::ZMQCMan, pid::Int, config::WorkerConfig)
send_data(get(config.userdata)[:zid], CONTROL_MSG, KILL_MSG)
(r_s, w_s) = get(config.userdata)[:streams]
close(r_s)
close(w_s)
# remove from our map
delete!(manager.map_zmq_julia, get(config.userdata)[:zid])
nothing
end
function print_worker_stdout(io, pid)
@schedule while !eof(io)
line = readline(io)
println("\tFrom worker $(pid):\t$line")
end
end

View File

@@ -0,0 +1,4 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
include("ZMQCM.jl")
start_broker()

View File

@@ -0,0 +1,11 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
include("ZMQCM.jl")
# @spawn run(`julia broker.jl`)
start_master(parse(Int,ARGS[1]))
resp = pmap(x -> myid() *2, [1:nworkers()])
println(resp)

View File

@@ -0,0 +1,5 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
include("ZMQCM.jl")
start_worker(parse(Int,ARGS[1]), ARGS[2])

View File

@@ -0,0 +1,12 @@
This is a simple proof-of-concept that uses UNIX domain sockets as transport.
All commands must be run from `examples/clustermanager/simple` directory
Start a Julia REPL and type:
include("UnixDomainCM.jl")
addprocs(UnixDomainCM(4)) # start with four workers
Alternatively, head.jl, a test script could be run. It just launches the requested number of workers,
executes a simple command on all of them and exits.
julia head.jl 4

View File

@@ -0,0 +1,92 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
import Base: launch, manage, connect, exit
mutable struct UnixDomainCM <: ClusterManager
np::Integer
end
function launch(manager::UnixDomainCM, params::Dict, launched::Array, c::Condition)
# println("launch $(manager.np)")
cookie = Base.cluster_cookie()
for i in 1:manager.np
sockname = tempname()
try
cmd = `$(params[:exename]) --startup-file=no $(@__FILE__) udwrkr $sockname $cookie`
io, pobj = open(cmd, "r")
wconfig = WorkerConfig()
wconfig.userdata = Dict(:sockname=>sockname, :io=>io, :process=>pobj)
push!(launched, wconfig)
notify(c)
catch e
println(e)
end
end
end
function connect(manager::UnixDomainCM, pid::Int, config::WorkerConfig)
if myid() == 1
# println("connect_m2w")
config.connect_at = get(config.userdata)[:sockname] # This will be useful in the worker-to-worker connection setup.
print_worker_stdout(get(config.userdata)[:io], pid)
else
# println("connect_w2w")
sockname = get(config.connect_at)
config.userdata = Dict{Symbol, Any}(:sockname=>sockname)
end
t = time()
while true
try
address = get(config.userdata)[:sockname]
if isa(address, Tuple)
sock = connect(address...)
else
sock = connect(address)
end
return (sock, sock)
catch e
if (time() - t) > 30.0
rethrow(e)
else
sleep(0.1)
end
end
end
end
# WORKER
function start_worker(sockname, cookie)
Base.init_worker(cookie, UnixDomainCM(0))
srvr = listen(sockname)
while true
sock = accept(srvr)
Base.process_messages(sock, sock)
end
end
function manage(manager::UnixDomainCM, id::Int, config::WorkerConfig, op)
# Does not seem to be required, filesystem entry cleanup is happening automatically on process exit
# if op == :deregister
# try
# rm(get(config.userdata)[:sockname])
# end
# end
nothing
end
function print_worker_stdout(io, pid)
@schedule while !eof(io)
line = readline(io)
println("\tFrom worker $(pid):\t$line")
end
end
if (length(ARGS) > 0) && (ARGS[1] == "udwrkr")
# script has been launched as a worker
start_worker(ARGS[2], ARGS[3])
end

View File

@@ -0,0 +1,7 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
include("UnixDomainCM.jl")
addprocs(UnixDomainCM(parse(Int,ARGS[1])))
resp = pmap(x -> myid() *2, [1:nworkers()])
println(resp)

View File

@@ -0,0 +1,12 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
cmanpath = joinpath(dirname(@__FILE__), "UnixDomainCM.jl")
include(cmanpath)
npids = addprocs(UnixDomainCM(2))
assert(length(npids) == 2)
test_pids = [remotecall_fetch(myid, x) for x in npids]
assert(npids == test_pids)
rmprocs(npids; waitfor=1.0)
exit(0)

View File

@@ -0,0 +1,34 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
import Base: put!, wait, isready, take!, fetch
mutable struct DictChannel <: AbstractChannel
d::Dict
cond_take::Condition # waiting for data to become available
DictChannel() = new(Dict(), Condition())
end
function put!(D::DictChannel, k, v)
D.d[k] = v
notify(D.cond_take)
D
end
function take!(D::DictChannel, k)
v=fetch(D,k)
delete!(D.d, k)
v
end
isready(D::DictChannel) = length(D.d) > 1
isready(D::DictChannel, k) = haskey(D.d,k)
function fetch(D::DictChannel, k)
wait(D,k)
D.d[k]
end
function wait(D::DictChannel, k)
while !isready(D, k)
wait(D.cond_take)
end
end

View File

@@ -0,0 +1,2 @@
/embedding
/embedding-debug

View File

@@ -0,0 +1,53 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
# This Makefile template requires the following variables to be set
# in the environment or on the command-line:
# JULIA: path to julia[.exe] executable
# BIN: binary build directory
ifndef JULIA
$(error "Please pass JULIA=[path of target julia binary], or set as environment variable!")
endif
ifndef BIN
$(error "Please pass BIN=[path of build directory], or set as environment variable!")
endif
#=============================================================================
# this source directory where embedding.c is located
SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
# get the executable suffix, if any
EXE := $(suffix $(abspath $(JULIA)))
# get compiler and linker flags. (see: `contrib/julia-config.jl`)
JULIA_CONFIG := $(JULIA) -e 'include(joinpath(JULIA_HOME, Base.DATAROOTDIR, "julia", "julia-config.jl"))' --
CPPFLAGS_ADD :=
CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags)
LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs)
DEBUGFLAGS += -g
#=============================================================================
release: $(BIN)/embedding$(EXE)
debug: $(BIN)/embedding-debug$(EXE)
$(BIN)/embedding$(EXE): $(SRCDIR)/embedding.c
$(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS)
$(BIN)/embedding-debug$(EXE): $(SRCDIR)/embedding.c
$(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS)
check: $(BIN)/embedding$(EXE)
$(JULIA) $(SRCDIR)/embedding-test.jl $<
@echo SUCCESS
clean:
-rm -f $(BIN)/embedding-debug$(EXE) $(BIN)/embedding$(EXE)
.PHONY: release debug clean check
# Makefile debugging trick:
# call print-VARIABLE to see the runtime value of any variable
print-%:
@echo '$*=$($*)'

View File

@@ -0,0 +1,22 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
# tests the output of the embedding example is correct
using Base.Test
@test length(ARGS) == 1
@testset "embedding example" begin
stdout = Pipe()
stderr = Pipe()
p = spawn(pipeline(Cmd(ARGS), stdin=DevNull, stdout=stdout, stderr=stderr))
close(stdout.in)
close(stderr.in)
stdout_task = @async readlines(stdout)
stderr = readstring(stderr)
@test stderr == "MethodError: no method matching this_function_has_no_methods()\n"
@test success(p)
lines = wait(stdout_task)
@test length(lines) == 9
@test parse(Float64, lines[1]) sqrt(2)
@test lines[8] == "called bar"
@test lines[9] == "calling new bar"
end

View File

@@ -0,0 +1,153 @@
// This file is a part of Julia. License is MIT: https://julialang.org/license
#include <julia.h>
#include <stdio.h>
#include <math.h>
#ifdef _OS_WINDOWS_
__declspec(dllexport) __cdecl
#endif
double my_c_sqrt(double x)
{
return sqrt(x);
}
jl_value_t *checked_eval_string(const char* code)
{
jl_value_t *result = jl_eval_string(code);
if (jl_exception_occurred()) {
// none of these allocate, so a gc-root (JL_GC_PUSH) is not necessary
jl_call2(jl_get_function(jl_base_module, "showerror"),
jl_stderr_obj(),
jl_exception_occurred());
jl_printf(jl_stderr_stream(), "\n");
jl_atexit_hook(1);
exit(1);
}
assert(result && "Missing return value but no exception occurred!");
return result;
}
int main()
{
jl_init();
{
// Simple running of Julia code
checked_eval_string("println(sqrt(2.0))");
}
{
// Accessing the return value
jl_value_t *ret = checked_eval_string("sqrt(2.0)");
double retDouble = jl_unbox_float64(ret);
printf("sqrt(2.0) in C: %e\n", retDouble);
fflush(stdout);
}
{
// Same as above but with function handle (more flexible)
jl_function_t *func = jl_get_function(jl_base_module, "sqrt");
jl_value_t* argument = jl_box_float64(2.0);
jl_value_t* ret = jl_call1(func, argument);
double retDouble = jl_unbox_float64(ret);
printf("sqrt(2.0) in C: %e\n", retDouble);
fflush(stdout);
}
{
// 1D arrays
jl_value_t* array_type = jl_apply_array_type((jl_value_t*)jl_float64_type, 1);
jl_array_t* x = jl_alloc_array_1d(array_type, 10);
// JL_GC_PUSH* is required here to ensure that `x` is not deleted before
// (aka, is gc-rooted until) the program reaches the corresponding JL_GC_POP()
JL_GC_PUSH1(&x);
double* xData = jl_array_data(x);
size_t i;
for (i = 0; i < jl_array_len(x); i++)
xData[i] = i;
jl_function_t *func = jl_get_function(jl_base_module, "reverse!");
jl_call1(func, (jl_value_t*) x);
printf("x = [");
for (i = 0; i < jl_array_len(x); i++)
printf("%e ", xData[i]);
printf("]\n");
fflush(stdout);
JL_GC_POP();
}
{
// Defining a Julia function and calling it
checked_eval_string("my_func(x) = 2 * x");
jl_function_t *func = jl_get_function(jl_current_module, "my_func");
jl_value_t* arg = jl_box_float64(5.0);
double ret = jl_unbox_float64(jl_call1(func, arg));
printf("my_func(5.0) = %f\n", ret);
fflush(stdout);
}
{
// Calling a C function from Julia (from C)
// in a shared library (exported, by name)
checked_eval_string("println( ccall(:my_c_sqrt, Float64, (Float64,), 2.0) )");
// or via a pointer
jl_value_t *call_by_ptr = checked_eval_string(
"my_c_sqrt -> println( ccall(my_c_sqrt, Float64, (Float64,), 2.0) )");
jl_call1(call_by_ptr, jl_box_voidpointer(my_c_sqrt));
}
{
// Handling exceptions gracefully
jl_value_t *f = checked_eval_string("function this_function_has_no_methods end");
jl_call0(f);
if (jl_exception_occurred()) {
jl_call2(jl_get_function(jl_base_module, "showerror"),
jl_stderr_obj(),
jl_exception_occurred());
jl_printf(jl_stderr_stream(), "\n");
}
}
{
// Creating and using a native C function handle
// to a Julia function signature
checked_eval_string(
"function bar()\n"
" println(\"called bar\")\n"
" random_return_value = 42\n"
"end");
checked_eval_string(
"function bar_from_c()\n"
" bar()\n"
" nothing\n"
"end");
typedef void (*Func_VOID__VOID)(void);
jl_value_t *pbar = jl_eval_string("cfunction(bar_from_c, Void, ())");
Func_VOID__VOID bar = (Func_VOID__VOID)jl_unbox_voidpointer(pbar);
bar();
checked_eval_string("bar() = println(\"calling new bar\")");
bar();
}
int ret = 0;
jl_atexit_hook(ret);
return ret;
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,130 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
module LRUExample
# An LRU (Least Recently Used) cache is an associative data structure which
# maintains its contents in an order such that the most recently used item
# is at the beginning of the structure, and the least recently used at the end.
#
# This file specifies two types of LRU caches, both with and without a size
# limit. BoundedLRU has a limit and evicts the LRU item if a new item is added
# after that bound is reached. UnboundedLRU does not have a maximum size, but
# can be used as a basis for more complex LRUs.
#
# LRUs should follow the interfaces for general collections, indexable
# collections, and associative collections.
# The standard implementation of an LRU backs a hash table with a doubly-linked
# list for O(1) operations when reordering on access and eviction. The Julia
# implementation instead backs the table with a Vector. For moderately-sized
# collections, the difference in performance is small, and this implmentation
# is simpler and easier to understand.
import Base.isempty, Base.length, Base.sizeof
import Base.start, Base.next, Base.done
import Base.haskey, Base.get
import Base.setindex!, Base.getindex, Base.delete!, Base.empty!
import Base.show
abstract type LRU{K,V} <: Associative{K,V} end
# Default cache size
const __MAXCACHE = 1024
mutable struct CacheItem{K,V}
k::K
v::V
end
mutable struct UnboundedLRU{K,V} <: LRU{K,V}
ht::Dict
q::Vector{CacheItem}
UnboundedLRU{K,V}() where {K,V} = new(Dict(), similar(Array{CacheItem}(1), 0))
end
UnboundedLRU() = UnboundedLRU{Any, Any}()
mutable struct BoundedLRU{K,V} <: LRU{K,V}
ht::Dict
q::Vector{CacheItem}
maxsize::Int
BoundedLRU{K,V}(m) where {K,V} = new(Dict(), similar(Array{CacheItem}(1), 0), m)
BoundedLRU{K,V}() where {K,V} = BoundedLRU(__MAXCACHE)
end
BoundedLRU(m) = BoundedLRU{Any, Any}(m)
BoundedLRU() = BoundedLRU{Any, Any}()
## collections ##
isempty(lru::LRU) = isempty(lru.q)
length(lru::LRU) = length(lru.q)
## associative ##
# Should this check count as an access?
haskey(lru::LRU, key) = haskey(lru.ht, key)
get(lru::LRU, key, default) = haskey(lru, key) ? lru[key] : default
function empty!(lru::LRU)
empty!(lru.ht)
empty!(lru.q)
end
show(io::IO, lru::UnboundedLRU) = print(io,"UnboundedLRU()")
show(io::IO, lru::BoundedLRU) = print(io,"BoundedLRU($(lru.maxsize))")
## indexable ##
# Method to do the second, slow lookup in the list with early return.
function locate(q, x)
for i = 1:length(q)
if q[i] == x
return i
end
end
error("Item not found.")
end
function getindex(lru::LRU, key)
item = lru.ht[key]
idx = locate(lru.q, item)
splice!(lru.q, idx)
unshift!(lru.q, item)
item.v
end
function setindex!(lru::LRU, v, key)
if haskey(lru, key)
item = lru.ht[key]
idx = locate(lru.q, item)
item.v = v
splice!(lru.q, idx)
else
item = CacheItem(key, v)
lru.ht[key] = item
end
unshift!(lru.q, item)
end
# Eviction
function setindex!{V,K}(lru::BoundedLRU, v::V, key::K)
invoke(setindex!, Tuple{LRU,V,K}, lru, v, key)
nrm = length(lru) - lru.maxsize
for i in 1:nrm
rm = pop!(lru.q)
delete!(lru.ht, rm.k)
end
end
## associative ##
function delete!(lru::LRU, key)
item = lru.ht[key]
idx = locate(lru.q, item)
delete!(lru.ht, key)
delete!(lru.q, idx)
end
end # module

View File

@@ -0,0 +1,47 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
using .LRUExample
TestLRU = LRUExample.UnboundedLRU{String, String}()
TestBLRU = LRUExample.BoundedLRU{String, String}(1000)
get_str(i) = String(vcat(map(x->[x>>4; x&0x0F], reinterpret(UInt8, [Int32(i)]))...))
isbounded{L<:LRUExample.LRU}(::Type{L}) = any(map(n->n==:maxsize, fieldnames(L)))
isbounded{L<:LRUExample.LRU}(l::L) = isbounded(L)
nmax = round.(Int, logspace(2, 5, 4))
function lrutest()
#println("LRU consistency tests")
for lru in (TestLRU,TestBLRU)
for n in nmax
empty!(lru)
#@printf(" %s, %d items\n", lru, n)
#print(" Simple eviction: ")
for i in 1:n
str = get_str(i)
lru[str] = str
@assert lru.q[1].v == str
if isbounded(lru) && length(lru) >= lru.maxsize
tailstr = get_str(i-lru.maxsize+1)
@assert lru.q[end].v == tailstr
end
end
#println("pass")
#print(" Lookup, random access: ")
for i in 1:n
str = get_str(rand(1:n))
if haskey(lru, str) # the bounded LRUs can have cache misses
blah = lru[str]
@assert lru.q[1].v == blah
end
end
#println("pass")
end
empty!(lru)
end
end
lrutest()

View File

@@ -0,0 +1,52 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
ndgrid(v::AbstractVector) = copy(v)
function ndgrid{T}(v1::AbstractVector{T}, v2::AbstractVector{T})
m, n = length(v1), length(v2)
v1 = reshape(v1, m, 1)
v2 = reshape(v2, 1, n)
(repmat(v1, 1, n), repmat(v2, m, 1))
end
function ndgrid_fill(a, v, s, snext)
for j = 1:length(a)
a[j] = v[div(rem(j-1, snext), s)+1]
end
end
function ndgrid{T}(vs::AbstractVector{T}...)
n = length(vs)
sz = map(length, vs)
out = ntuple(i->Array{T}(sz), n)
s = 1
for i=1:n
a = out[i]::Array
v = vs[i]
snext = s*size(a,i)
ndgrid_fill(a, v, s, snext)
s = snext
end
out
end
meshgrid(v::AbstractVector) = meshgrid(v, v)
function meshgrid{T}(vx::AbstractVector{T}, vy::AbstractVector{T})
m, n = length(vy), length(vx)
vx = reshape(vx, 1, n)
vy = reshape(vy, m, 1)
(repmat(vx, m, 1), repmat(vy, 1, n))
end
function meshgrid{T}(vx::AbstractVector{T}, vy::AbstractVector{T},
vz::AbstractVector{T})
m, n, o = length(vy), length(vx), length(vz)
vx = reshape(vx, 1, n, 1)
vy = reshape(vy, m, 1, 1)
vz = reshape(vz, 1, 1, o)
om = ones(Int, m)
on = ones(Int, n)
oo = ones(Int, o)
(vx[om, :, oo], vy[:, on, oo], vz[om, on, :])
end

View File

@@ -0,0 +1,28 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
# n-queens (nqueens) solver, for nsquaresx-by-nsquaresy board
struct Queen
x::Int
y::Int
end
hitshorz(queena, queenb) = queena.x == queenb.x
hitsvert(queena, queenb) = queena.y == queenb.y
hitsdiag(queena, queenb) = abs(queena.x - queenb.x) == abs(queena.y - queenb.y)
hitshvd(qa, qb) = hitshorz(qa, qb) || hitsvert(qa, qb) || hitsdiag(qa, qb)
hitsany(testqueen, queens) = any(q -> hitshvd(testqueen, q), queens)
function trysolve(nsquaresx, nsquaresy, nqueens, presqueens = ())
nqueens == 0 && return presqueens
for xsquare in 1:nsquaresx
for ysquare in 1:nsquaresy
testqueen = Queen(xsquare, ysquare)
if !hitsany(testqueen, presqueens)
tryqueens = (presqueens..., testqueen)
maybesol = trysolve(nsquaresx, nsquaresy, nqueens - 1, tryqueens)
maybesol !== nothing && return maybesol
end
end
end
return nothing
end

View File

@@ -0,0 +1,4 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
x="println(\"# This file is a part of Julia. License is MIT: https://julialang.org/license\\n\\nx=\$(repr(x))\\n\$x\")"
println("# This file is a part of Julia. License is MIT: https://julialang.org/license\n\nx=$(repr(x))\n$x")

View File

@@ -0,0 +1,61 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
function add_method(gf, an, at, body)
argexs = [Expr(Symbol("::"), an[i], at[i]) for i=1:length(an)]
def = quote
let __F__=($gf)
function __F__($(argexs...))
$body
end
end
end
eval(def)
end
macro staged(fdef)
if !isa(fdef,Expr) || fdef.head !== :function
error("@staged: expected method definition")
end
fname = fdef.args[1].args[1]
argspec = fdef.args[1].args[2:end]
argnames = map(x->(isa(x,Expr) ? x.args[1] : x), argspec)
qargnames = map(x->Expr(:quote,x), argnames)
fbody = fdef.args[2]
@gensym gengf argtypes expander genbody
quote
let ($gengf)
global ($fname) # should be "outer"
local ($expander)
function ($expander)($(argnames...))
$fbody
end
($gengf)() = 0 # should be initially empty GF
function ($fname)($(argspec...))
($argtypes) = typeof(tuple($(argnames...)))
if !method_exists($gengf, $argtypes)
($genbody) = apply(($expander), ($argtypes))
add_method($gengf, Any[$(qargnames...)],
$argtypes, $genbody)
end
return ($gengf)($(argnames...))
end
end
end
end
# example
@staged function nloops(dims::Tuple)
names = map(x->gensym(), dims)
ex = quote
println([$(names...)])
end
for i = 1:length(dims)
ex = quote
for $(names[i]) in dims[$i]
$ex
end
end
end
ex
end

View File

@@ -0,0 +1,45 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
module Time
export TimeDelta
import Base.show, Base.+, Base.-, Base.convert, Base.promote_rule
struct TimeDelta{p}
v::Int64
end
const PREFIXES = [
"yocto", "zepto", "atto", "femto", "pico", "nano", "micro", "milli",
"", "kilo", "mega", "giga", "tera", "peta", "exa", "zetta", "yotta",
]
const ZERO_INDEX = 9
const MAX_INDEX = 17
function show{p}(io::IO, x::TimeDelta{p})
k = max(1,min(MAX_INDEX,fld(p,3)+ZERO_INDEX))
r = p-3(k-ZERO_INDEX)
prefix = PREFIXES[k]
if r == 0
s = x.v == 1 ? "" : "s"
print(io, "$(x.v) $(prefix)second$s")
elseif r > 0
print(io, "$(x.v*10^r) $(prefix)seconds")
else
print(io, "$(x.v/10^-r) $(prefix)seconds")
end
end
convert{p,q}(::Type{TimeDelta{p}}, x::TimeDelta{q}) =
TimeDelta{p}(p <= q ? x.v*10^(q-p) : div(x.v,10^(p-q)))
promote_rule{p,q}(::Type{TimeDelta{p}}, ::Type{TimeDelta{q}}) = TimeDelta{min(p,q)}
-{p}(x::TimeDelta{p}) = TimeDelta{p}(-x.v)
+{p}(x::TimeDelta{p}, y::TimeDelta{p}) = TimeDelta{p}(x.v+y.v)
-{p}(x::TimeDelta{p}, y::TimeDelta{p}) = TimeDelta{p}(x.v-y.v)
+(x::TimeDelta, y::TimeDelta) = +(promote(x,y)...)
-(x::TimeDelta, y::TimeDelta) = -(promote(x,y)...)
end # module

View File

@@ -0,0 +1,154 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
module TypeTrees
##
# Generate a text graphic of Julia modules type tree
##
struct Binding
mod::Module
sym::Symbol
end
Binding(tn::TypeName) = Binding(tn.module, tn.name)
Base.isless(a::Binding, b::Binding) = isless(a.sym, b.sym)
# The node type holds the type of the current node and a dict of subtypes
struct TTNode
typ # ::Type
subtypes::Dict{Binding, TTNode}
TTNode(t::ANY) = new(t, Dict{Binding, TTNode}())
end
# Add a node to a dict if not added
function add_ttnode(subtypes::Dict{Binding, TTNode}, sname::Binding, tnode::TTNode)
return get!(subtypes, sname, tnode)
end
function add_ttnode(subtypes::Dict{Binding, TTNode}, sname::Binding, t::Type)
return get!(subtypes, sname, TTNode(t))
end
# Store a type and its type hierarchy chain
# Recurse till we reach the top level type
function store_type(sname::Binding, t::Union)
suptype = Union
tnode = TTNode(t)
# store unions under Union type
subtypes = store_type(Binding(suptype.name), suptype)
add_ttnode(subtypes, sname, tnode)
store_union(sname, tnode, t)
# unions are also in a sense related to the types of their components
return tnode.subtypes
end
function store_union(sname::Binding, tnode::TTNode, t::ANY)
t = Base.unwrap_unionall(t)
if isa(t, Union)
store_union(sname, tnode, t.a)
store_union(sname, tnode, t.b)
elseif isa(t, DataType)
binding = Binding(t.name)
subtypes = store_type(binding, t)
add_ttnode(subtypes, sname, tnode)
end
nothing
end
function store_type(sname::Binding, t::UnionAll)
suptype = Base.unwrap_unionall(t)
binding = isa(suptype, DataType) ? Binding(suptype.name) : Binding(Main, string(suptype::Union))
subtypes = store_type(binding, suptype)
tnode = add_ttnode(subtypes, sname, t)
return tnode.subtypes
end
function store_type(sname::Binding, t::DataType)
suptype = supertype(t)
subtypes = (suptype != t) ? store_type(Binding(suptype.name), suptype) : types_tree
tnode = add_ttnode(subtypes, sname, t)
return tnode.subtypes
end
# examine all symbols in module and store those that are types
function store_all_from(m::Module)
for s in names(m, true)
if isdefined(m, s) && !Base.isdeprecated(m, s)
t = getfield(m, s)
if isa(t, Type) && t !== Union{}
store_type(Binding(m, s), t)
elseif isa(t, Module) && module_name(t) === s && module_parent(t) === m && t !== m
store_all_from(t)
end
end
end
end
type_props(typ) = ""
type_props(typ::DataType) = string("<<",
typ.abstract ? " abstract" : " concrete",
typ.mutable ? " mutable" : " immutable",
typ.layout != C_NULL ? string(
Base.datatype_pointerfree(typ) ? " pointerfree" : "",
Base.datatype_haspadding(typ) ? " haspadding" : "",
" nfields:", Core.nfields(typ),
" size:", typ.size,
", align:", Base.datatype_alignment(typ)) : "",
" >>")
function print_tree(subtypes::Dict{Binding, TTNode}, pfx::String="")
for b in sort!(collect(keys(subtypes)))
v = subtypes[b]
ishidden = unsafe_load(Base.unsafe_convert(Ptr{UInt8}, b.sym)) == UInt8('#')
if ishidden && supertype(v.typ) === Function
continue
end
if b.mod === Main
n = string(b.sym)
elseif !isa(v.typ, DataType) || v.typ.name.module != b.mod || v.typ.name.name != b.sym
n_io = IOBuffer()
print(n_io, b.mod, '.', b.sym)
ua = v.typ
if isa(ua, UnionAll)
print(n_io, "{")
while true
print(n_io, ua.var)
ua = ua.body
if isa(ua, UnionAll)
print(n_io, ", ")
else
break
end
end
print(n_io, "}")
end
n = String(take!(n_io))
else
n = string(v.typ)
end
if n == string(v.typ)
println(pfx, "+- ", n, " ", type_props(v.typ))
else
println(pfx, "+- ", n, " = ", v.typ, " ", type_props(v.typ))
end
v.typ === Function && println(pfx, ". ## hiding implicit Function subtypes ##")
print_tree(v.subtypes, pfx * ". ")
end
end
# TODO: optionally take module names in command line
# TODO: option to list subtrees of type tree, or other symbol types
const types_tree = Dict{Binding, TTNode}()
store_all_from(Main)
# print_tree(types_tree)
end # module
if !isinteractive()
TypeTrees.print_tree(TypeTrees.types_tree)
end

View File

@@ -0,0 +1,86 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license
# wordcount.jl
#
# Implementation of parallelized "word-count" of a text, inspired by the
# Hadoop WordCount example. Uses @spawn and fetch() to parallelize
# the "map" task. Reduce is currently done single-threaded.
#
# To run in parallel on a string stored in variable `text`:
# julia -p <N>
# julia> require("<julia_doc_dir>/examples/wordcount.jl")
# julia> ...(define text)...
# julia> counts=parallel_wordcount(text)
#
# Or to run on a group of files, writing results to an output file:
# julia -p <N>
# julia> require("<julia_doc_dir>/examples/wordcount.jl")
# julia> wordcount_files("/tmp/output.txt", "/tmp/input1.txt","/tmp/input2.txt",...)
# "Map" function.
# Takes a string. Returns a Dict with the number of times each word
# appears in that string.
function wordcount(text)
words=split(text,[' ','\n','\t','-','.',',',':',';'];keep=false)
counts=Dict()
for w = words
counts[w]=get(counts,w,0)+1
end
return counts
end
# "Reduce" function.
# Takes a collection of Dicts in the format returned by wordcount()
# Returns a Dict in which words that appear in multiple inputs
# have their totals added together.
function wcreduce(wcs)
counts=Dict()
for c in wcs, (k,v) in c
counts[k] = get(counts,k,0)+v
end
return counts
end
# Splits input string into nprocs() equal-sized chunks (last one rounds up),
# and @spawns wordcount() for each chunk to run in parallel. Then fetch()s
# results and performs wcreduce().
function parallel_wordcount(text)
lines=split(text,'\n';keep=false)
np=nprocs()
unitsize=ceil(length(lines)/np)
wcounts=[]
rrefs=[]
# spawn procs
for i=1:np
first=unitsize*(i-1)+1
last=unitsize*i
if last>length(lines)
last=length(lines)
end
subtext=join(lines[Int(first):Int(last)],"\n")
push!(rrefs, @spawn wordcount( subtext ) )
end
# fetch results
while length(rrefs)>0
push!(wcounts,fetch(pop!(rrefs)))
end
# reduce
count=wcreduce(wcounts)
return count
end
# Takes the name of a result file, and a list of input file names.
# Combines the contents of all files, then performs a parallel_wordcount
# on the resulting string. Writes the results to result_file.
function wordcount_files(result_file,inputs...)
text = ""
for file in inputs
text *= readstring(file)
end
wc = parallel_wordcount(text)
open(result_file,"w") do f
for (k,v) in wc
println(f, k,"=",v)
end
end
end