AWS CLI で S3 からダウンロードしたメモ。
$ export LANG=C
$ time dd if=/dev/urandom of=10gb.dat bs=1M count=10240
10240+0 records in
10240+0 records out
10737418240 bytes (11 GB) copied, 57.8513 s, 186 MB/s
real 0m57.853s
user 0m0.008s
sys 0m57.843s
$ time aws s3 cp 10gb.dat s3://cp-src/dltest/10gb.dat
upload: ./10gb.dat to s3://cp-src/dltest/10gb.dat
real 0m42.508s
user 0m59.424s
sys 0m43.174s
$ time seq -w 1 10 | xargs -t -P10 -I{} aws s3 cp s3://cp-src/dltest/10gb.dat s3://cp-src/dltest/{}.dat
real 0m48.172s
user 0m53.549s
sys 0m9.533s
$ time seq -w 1 10 | xargs -t -P10 -I{} aws s3 cp s3://cp-src/dltest/{}.dat .
real 5m27.284s
user 8m33.225s
sys 4m46.696s
$ aws configure set default.s3.multipart_threshold 1024MB
$ aws configure set default.s3.multipart_chunksize 512MB
$ aws configure set default.s3.max_concurrent_requests 1000
$ time seq -w 1 10 | xargs -t -P10 -I{} aws s3 cp s3://cp-src/dltest/{}.dat .
real 5m26.150s
user 7m11.524s
sys 4m37.971s
