Commit 0a863fdf authored by Brian Groenke's avatar Brian Groenke
Browse files

Update ensemble simulation runs

parent 7e5754be
......@@ -375,6 +375,12 @@ git-tree-sha1 = "31186e61936fbbccb41d809ad4338c9f7addf7ae"
uuid = "754358af-613d-5f8d-9788-280bf1605d4c"
version = "0.2.0"
[[Dagger]]
deps = ["Colors", "Distributed", "LinearAlgebra", "MemPool", "Profile", "Random", "Requires", "Serialization", "SharedArrays", "SparseArrays", "Statistics", "StatsBase", "TableOperations", "Tables", "UUIDs"]
git-tree-sha1 = "8a0fc297de557c3272fceb7fb5855ec4ea85929c"
uuid = "d58978e5-989f-55fb-8d15-ea34adc7bf54"
version = "0.12.5"
[[DataAPI]]
git-tree-sha1 = "ee400abb2298bd13bfc3df1c412ed228061a2385"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
......@@ -648,6 +654,12 @@ git-tree-sha1 = "0f5e8d0cb91a6386ba47bd1527b240bd5725fbae"
uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
version = "0.9.10"
[[FileTrees]]
deps = ["AbstractTrees", "Dagger", "FilePathsBase", "Glob"]
git-tree-sha1 = "03a475ac80fe9f56eeed72388c9c6731edcf74d9"
uuid = "72696420-646e-6120-6e77-6f6420746567"
version = "0.3.2"
[[FileWatching]]
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
......@@ -815,6 +827,11 @@ git-tree-sha1 = "7bf67e9a481712b3dbe9cb3dac852dc4b1162e02"
uuid = "7746bdde-850d-59dc-9ae8-88ece973131d"
version = "2.68.3+0"
[[Glob]]
git-tree-sha1 = "4df9f7e06108728ebf00a0a11edee4b29a482bb2"
uuid = "c27321d9-0574-5035-807b-f59d2c89b15c"
version = "1.3.0"
[[GlobalSensitivity]]
deps = ["Distributions", "FFTW", "ForwardDiff", "KernelDensity", "LinearAlgebra", "Parameters", "QuasiMonteCarlo", "Random", "RecursiveArrayTools", "Statistics", "StatsBase", "Trapz"]
git-tree-sha1 = "c737e4e6baabac7eb9273ba0b047d80a8a560bc4"
......@@ -1349,6 +1366,12 @@ git-tree-sha1 = "75a54abd10709c01f1b86b84ec225d26e840ed58"
uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27"
version = "0.5.0"
[[MemPool]]
deps = ["DataStructures", "Distributed", "Mmap", "Random", "Serialization", "Sockets"]
git-tree-sha1 = "334dd674df3fb0f0ed453b3384c3aabe22be29b0"
uuid = "f9f48841-c794-520a-933b-121f7ba6ed94"
version = "0.3.6"
[[Memento]]
deps = ["Dates", "Distributed", "JSON", "Serialization", "Sockets", "Syslogs", "Test", "TimeZones", "UUIDs"]
git-tree-sha1 = "19650888f97362a2ae6c84f0f5f6cda84c30ac38"
......
......@@ -15,6 +15,7 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
FileTrees = "72696420-646e-6120-6e77-6f6420746567"
Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
GalacticOptim = "a75be94c-b780-496d-a8a9-0878b188d577"
......
outs:
- md5: 708909e98207a79d5f85623601fcb994.dir
size: 16097652982
- md5: 415f6293dc704782ac908e749d40f4db.dir
size: 703446386
nfiles: 12
path: exp_raw
......@@ -4,11 +4,11 @@ stages:
cmd: julia --project=. scripts/simulations/samoylov/sahole_cryogrid_sim.jl
deps:
- path: Manifest.toml
md5: 786459bc05ad1fde2d910be0e9dc32e5
size: 92750
md5: e9c95a100f80e2f37d5b876f79c22b68
size: 93541
- path: data/exp_raw
md5: 708909e98207a79d5f85623601fcb994.dir
size: 16097652982
md5: 415f6293dc704782ac908e749d40f4db.dir
size: 703446386
nfiles: 12
- path: scripts/simulations/samoylov/sahole_cryogrid_sim.jl
md5: c31a2d6704450b3374eaf6c1aab19cf0
......@@ -29,11 +29,11 @@ stages:
cmd: julia --project=. scripts/preprocessing/preprocess_trends.jl
deps:
- path: Manifest.toml
md5: 786459bc05ad1fde2d910be0e9dc32e5
size: 92750
md5: e9c95a100f80e2f37d5b876f79c22b68
size: 93541
- path: data/exp_raw
md5: 708909e98207a79d5f85623601fcb994.dir
size: 16097652982
md5: 415f6293dc704782ac908e749d40f4db.dir
size: 703446386
nfiles: 12
- path: data/sims/sahole_cryogrid
md5: b88d98c54c7608ccc15cc914c844eb67.dir
......@@ -55,8 +55,8 @@ stages:
cmd: julia --project=. --threads=4 scripts/inference/Ts_trend_inference.jl
deps:
- path: Manifest.toml
md5: 786459bc05ad1fde2d910be0e9dc32e5
size: 92750
md5: e9c95a100f80e2f37d5b876f79c22b68
size: 93541
- path: data/exp_pro/trends
md5: 019b0976e4c4b67b9f88704be87b5f69.dir
size: 613236
......@@ -77,18 +77,18 @@ stages:
cmd: julia --project=. scripts/simulations/ensemble/run_cryogrid_ensemble.jl
deps:
- path: Manifest.toml
md5: 786459bc05ad1fde2d910be0e9dc32e5
size: 92750
md5: e9c95a100f80e2f37d5b876f79c22b68
size: 93541
- path: data/exp_raw
md5: 708909e98207a79d5f85623601fcb994.dir
size: 16097652982
md5: 415f6293dc704782ac908e749d40f4db.dir
size: 703446386
nfiles: 12
- path: scripts/simulations/ensemble/run_cryogrid_ensemble.jl
md5: b3aa750df78647a068b0d57164a1a889
size: 2358
md5: a32d80f21917efa936971e445f44c05a
size: 2794
- path: scripts/simulations/ensemble/setup_cryogrid_ensemble.jl
md5: 565da97abbb57b8babefc2ea8a79bc37
size: 2349
md5: 1398eada66c92879b2cda97aa84a603f
size: 1639
params:
params.yaml:
simulate_ensemble.dataset: samoylov
......@@ -98,6 +98,6 @@ stages:
simulate_ensemble.tstart_warmup: '1979-10-30'
outs:
- path: data/sims/ensemble_cryogrid/
md5: 4b2f740d9d7380164b07aad8c1aeecf2.dir
size: 54706643000
md5: 5a7bee9242c5937d35217cda2157374c.dir
size: 54692734162
nfiles: 1000
......@@ -13,7 +13,7 @@ workers = addprocs(exeflags="--project=.")
@everywhere begin
# define functions for ensemble
runfile(i) = datadir("sims", "ensemble_cryogrid", string(i; pad=Int(floor(log10(nsamples)))), savename("cg_ensemble", params, "nc"))
runfile(i) = datadir("sims", "ensemble_cryogrid", params["dataset"], string(i; pad=Int(floor(log10(nsamples)))), savename("cg_ensemble_daily", params, "nc"))
dim2geoarray(A::DimArray, name::Symbol) = GeoArray(A.data, (Z(ustrip.(dims(A,Z).val)), dims(A,Ti)), name=name)
generate_prob(prob, i, repeat) = remake(prob, p=p_prior())
function postprocess(sol, i)
......@@ -60,3 +60,11 @@ end
# ensprob = EnsembleProblem(prob, prob_func=prob_func, output_func=output_func)
# enssol = @time solve(ensprob, Euler(), EnsembleDistributed(), dt=dt, saveat=saveat, trajectories=8, callback=CFLStepLimiter(setup))
@info "[$(myid())] Ensemble runs finished"
# aggregate model outputs to annual products
outputtree = FileTree(datadir("sims", "ensemble_cryogrid"))
@time @sync @distributed for file in FileTrees.files(outputtree)
ncstack = GeoData.stack(FileTrees.path(file))
results = map(arr -> GeoArray(Data.aggregate_yearly(arr); name=GeoData.name(arr)), ncstack)
outfile = rename(file, replace(FileTrees.name(file), "cg_ensemble" => "cg_ensemble_annual"))
GeoData.write(FileTrees.path(outfile), NCDstack, results)
end
......@@ -6,6 +6,7 @@ using CryoGrid
using Dates
using DrWatson
using DimensionalData, NCDatasets, GeoData
using FileTrees
using Impute
using JSON3
using Logging
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment