#!/bin/bash ### --- ### name: 'SOMATIC_CNV_REF_BUILDER:SOMA_CNV_BUILD_REFERENCE (build_reference)' ### container: '292967571998.dkr.ecr.us-west-2.amazonaws.com/soma-cnv:20260604-3cb5ec5' ### outputs: ### - 'batch_reference.parquet' ### - 'sex_calls.csv' ### - 'feature_stats.csv' ### - 'versions.yml' ### ... set -e set -u NXF_DEBUG=${NXF_DEBUG:=0}; [[ $NXF_DEBUG > 1 ]] && set -x NXF_ENTRY=${1:-nxf_main} nxf_tree() { local pid=$1 declare -a ALL_CHILDREN while read P PP;do ALL_CHILDREN[$PP]+=" $P" done < <(ps -e -o pid= -o ppid=) pstat() { local x_pid=$1 local STATUS=$(2> /dev/null < /proc/$1/status grep -E 'Vm|ctxt') if [ $? = 0 ]; then local x_vsz=$(echo "$STATUS" | grep VmSize | awk '{print $2}' || echo -n '0') local x_rss=$(echo "$STATUS" | grep VmRSS | awk '{print $2}' || echo -n '0') local x_peak=$(echo "$STATUS" | grep -E 'VmPeak|VmHWM' | sed 's/^.*:\s*//' | sed 's/[\sa-zA-Z]*$//' | tr '\n' ' ' || echo -n '0 0') local x_pmem=$(awk -v rss=$x_rss -v mem_tot=$mem_tot 'BEGIN {printf "%.0f", rss/mem_tot*100*10}' || echo -n '0') local vol_ctxt=$(echo "$STATUS" | grep '\bvoluntary_ctxt_switches' | awk '{print $2}' || echo -n '0') local inv_ctxt=$(echo "$STATUS" | grep '\bnonvoluntary_ctxt_switches' | awk '{print $2}' || echo -n '0') cpu_stat[x_pid]="$x_pid $x_pmem $x_vsz $x_rss $x_peak $vol_ctxt $inv_ctxt" fi } pwalk() { pstat $1 for i in ${ALL_CHILDREN[$1]:=}; do pwalk $i; done } pwalk $1 } nxf_stat() { cpu_stat=() nxf_tree $1 declare -a sum=(0 0 0 0 0 0 0 0) local pid local i for pid in "${!cpu_stat[@]}"; do local row=(${cpu_stat[pid]}) [ $NXF_DEBUG = 1 ] && echo "++ stat mem=${row[*]}" for i in "${!row[@]}"; do if [ $i != 0 ]; then sum[i]=$((sum[i]+row[i])) fi done done [ $NXF_DEBUG = 1 ] && echo -e "++ stat SUM=${sum[*]}" for i in {1..7}; do if [ ${sum[i]} -lt ${cpu_peak[i]} ]; then sum[i]=${cpu_peak[i]} else cpu_peak[i]=${sum[i]} fi done [ $NXF_DEBUG = 1 ] && echo -e "++ stat PEAK=${sum[*]}\n" nxf_stat_ret=(${sum[*]}) } nxf_mem_watch() { set -o pipefail local pid=$1 local trace_file=.command.trace local count=0; declare -a cpu_stat=(0 0 0 0 0 0 0 0) declare -a cpu_peak=(0 0 0 0 0 0 0 0) local mem_tot=$(< /proc/meminfo grep MemTotal | awk '{print $2}') local timeout local DONE local STOP='' [ $NXF_DEBUG = 1 ] && nxf_sleep 0.2 && ps fx while true; do nxf_stat $pid if [ $count -lt 10 ]; then timeout=1; elif [ $count -lt 120 ]; then timeout=5; else timeout=30; fi read -t $timeout -r DONE || true [[ $DONE ]] && break if [ ! -e /proc/$pid ]; then [ ! $STOP ] && STOP=$(nxf_date) [ $(($(nxf_date)-STOP)) -gt 10000 ] && break fi count=$((count+1)) done printf "%s\n" \ "%mem=${nxf_stat_ret[1]}" \ "vmem=${nxf_stat_ret[2]}" \ "rss=${nxf_stat_ret[3]}" \ "peak_vmem=${nxf_stat_ret[4]}" \ "peak_rss=${nxf_stat_ret[5]}" \ "vol_ctxt=${nxf_stat_ret[6]}" \ "inv_ctxt=${nxf_stat_ret[7]}" >> "$trace_file" || >&2 echo "Error: Failed to append to file: $trace_file" } nxf_write_trace() { printf "%s\n" \ "nextflow.trace/v2" \ "realtime=$wall_time" \ "%cpu=$ucpu" \ "cpu_model=$cpu_model" \ "rchar=${io_stat1[0]}" \ "wchar=${io_stat1[1]}" \ "syscr=${io_stat1[2]}" \ "syscw=${io_stat1[3]}" \ "read_bytes=${io_stat1[4]}" \ "write_bytes=${io_stat1[5]}" >| "$trace_file" || >&2 echo "Error: Failed to write to file: $trace_file" } nxf_trace_mac() { local start_millis=$(nxf_date) /bin/bash -Ceuo pipefail .command.sh local end_millis=$(nxf_date) local wall_time=$((end_millis-start_millis)) local ucpu='' local cpu_model='' local io_stat1=('' '' '' '' '' '') nxf_write_trace } nxf_fd() { local FD=11 while [ -e /proc/$$/fd/$FD ]; do FD=$((FD+1)); done echo $FD } nxf_trace_linux() { local pid=$$ command -v ps &>/dev/null || { >&2 echo "Command 'ps' required by nextflow to collect task metrics cannot be found"; exit 1; } local num_cpus=$(< /proc/cpuinfo grep '^processor' -c) local cpu_model=$(< /proc/cpuinfo grep '^model name' | head -n 1 | awk 'BEGIN{FS="\t: "} { print $2 }') local tot_time0=$(grep '^cpu ' /proc/stat | awk '{sum=$2+$3+$4+$5+$6+$7+$8+$9; printf "%.0f",sum}') local cpu_time0=$(2> /dev/null < /proc/$pid/stat awk '{printf "%.0f", ($16+$17)*10 }' || echo -n 'X') local io_stat0=($(2> /dev/null < /proc/$pid/io sed 's/^.*:\s*//' | head -n 6 | tr '\n' ' ' || echo -n '0 0 0 0 0 0')) local start_millis=$(nxf_date) trap 'kill $mem_proc' ERR /bin/bash -Ceuo pipefail .command.sh & local task=$! mem_fd=$(nxf_fd) eval "exec $mem_fd> >(nxf_mem_watch $task)" local mem_proc=$! wait $task local end_millis=$(nxf_date) local tot_time1=$(grep '^cpu ' /proc/stat | awk '{sum=$2+$3+$4+$5+$6+$7+$8+$9; printf "%.0f",sum}') local cpu_time1=$(2> /dev/null < /proc/$pid/stat awk '{printf "%.0f", ($16+$17)*10 }' || echo -n 'X') local ucpu=$(awk -v p1=$cpu_time1 -v p0=$cpu_time0 -v t1=$tot_time1 -v t0=$tot_time0 -v n=$num_cpus 'BEGIN { pct=(p1-p0)/(t1-t0)*100*n; printf("%.0f", pct>0 ? pct : 0) }' ) local io_stat1=($(2> /dev/null < /proc/$pid/io sed 's/^.*:\s*//' | head -n 6 | tr '\n' ' ' || echo -n '0 0 0 0 0 0')) local i for i in {0..5}; do io_stat1[i]=$((io_stat1[i]-io_stat0[i])) done local wall_time=$((end_millis-start_millis)) [ $NXF_DEBUG = 1 ] && echo "+++ STATS %CPU=$ucpu TIME=$wall_time I/O=${io_stat1[*]}" printf "%s\n" \ "nextflow.trace/v2" \ "realtime=$wall_time" \ "%cpu=$ucpu" \ "cpu_model=$cpu_model" \ "rchar=${io_stat1[0]}" \ "wchar=${io_stat1[1]}" \ "syscr=${io_stat1[2]}" \ "syscw=${io_stat1[3]}" \ "read_bytes=${io_stat1[4]}" \ "write_bytes=${io_stat1[5]}" >| "$trace_file" || >&2 echo "Error: Failed to write to file: $trace_file" [ -e /proc/$mem_proc ] && eval "echo 'DONE' >&$mem_fd" || true wait $mem_proc 2>/dev/null || true while [ -e /proc/$mem_proc ]; do nxf_sleep 0.1; done } nxf_trace() { local trace_file=.command.trace touch $trace_file if [[ $(uname) = Darwin ]]; then nxf_trace_mac else nxf_trace_linux fi } # bash helper functions nxf_cp_retry() { local max_attempts=1 local timeout=10 local attempt=0 local exitCode=0 while (( $attempt < $max_attempts )) do if "$@" then return 0 else exitCode=$? fi if [[ $exitCode == 0 ]] then break fi nxf_sleep $timeout attempt=$(( attempt + 1 )) timeout=$(( timeout * 2 )) done } nxf_parallel() { IFS=$'\n' local cmd=("$@") local cpus=$(nproc 2>/dev/null || < /proc/cpuinfo grep '^process' -c) local max=$(if (( cpus>4 )); then echo 4; else echo $cpus; fi) local i=0 local pid=() ( set +u while ((i<${#cmd[@]})); do local copy=() for x in "${pid[@]}"; do # if the process exist, keep in the 'copy' array, otherwise wait on it to capture the exit code # see https://github.com/nextflow-io/nextflow/pull/4050 [[ -e /proc/$x ]] && copy+=($x) || wait $x done pid=("${copy[@]}") if ((${#pid[@]}>=$max)); then nxf_sleep 0.2 else eval "${cmd[$i]}" & pid+=($!) ((i+=1)) fi done for p in "${pid[@]}"; do wait $p done ) unset IFS } # aws helper for s5cmd nxf_s3_upload() { local name=$1 local s3path=$2 if [[ "$name" == - ]]; then local tmp=$(nxf_mktemp) cp /dev/stdin $tmp/$name /opt/s5cmd/bin/s5cmd --log error cp --storage-class STANDARD $tmp/$name "$s3path" elif [[ -d "$name" ]]; then /opt/s5cmd/bin/s5cmd --log error cp --storage-class STANDARD "$name/" "$s3path/$name/" else /opt/s5cmd/bin/s5cmd --log error cp --storage-class STANDARD "$name" "$s3path/$name" fi } nxf_s3_download() { local source=$1 local target=$2 echo " Downloading: $source" local file_name=$(basename $1) local is_dir=$(/opt/s5cmd/bin/s5cmd ls $source | grep -F "DIR ${file_name}/" -c) if [[ $is_dir == 1 ]]; then /opt/s5cmd/bin/s5cmd --log error cp "$source/*" "$target" else /opt/s5cmd/bin/s5cmd --log error cp "$source" "$target" fi } nxf_sleep() { sleep $1 2>/dev/null || sleep 1; } nxf_date() { local ts=$(date +%s%3N); if [[ ${#ts} == 10 ]]; then echo ${ts}000 elif [[ $ts == *%3N ]]; then echo ${ts/\%3N/000} elif [[ $ts == *3N ]]; then echo ${ts/3N/000} elif [[ ${#ts} == 13 ]]; then echo $ts else echo "Unexpected timestamp value: $ts"; exit 1 fi } nxf_env() { echo '============= task environment =============' env | sort | sed "s/\(.*\)AWS\(.*\)=\(.\{6\}\).*/\1AWS\2=\3xxxxxxxxxxxxx/" echo '============= task output ==================' } nxf_kill() { declare -a children while read P PP;do children[$PP]+=" $P" done < <(ps -e -o pid= -o ppid=) kill_all() { [[ $1 != $$ ]] && kill $1 2>/dev/null || true for i in ${children[$1]:=}; do kill_all $i; done } kill_all $1 } nxf_mktemp() { local base=${1:-/tmp} mkdir -p "$base" if [[ $(uname) = Darwin ]]; then mktemp -d $base/nxf.XXXXXXXXXX else TMPDIR="$base" mktemp -d -t nxf.XXXXXXXXXX fi } nxf_fs_copy() { local source=$1 local target=$2 local basedir=$(dirname $1) mkdir -p $target/$basedir cp -fRL $source $target/$basedir } nxf_fs_move() { local source=$1 local target=$2 local basedir=$(dirname $1) mkdir -p $target/$basedir mv -f $source $target/$basedir } nxf_fs_rsync() { rsync -rRl $1 $2 } nxf_fs_rclone() { rclone copyto $1 $2/$1 } nxf_fs_fcp() { fcp $1 $2/$1 } on_exit() { local last_err=$? local exit_status=${nxf_main_ret:=0} [[ ${exit_status} -eq 0 && ${nxf_unstage_ret:=0} -ne 0 ]] && exit_status=${nxf_unstage_ret:=0} [[ ${exit_status} -eq 0 && ${last_err} -ne 0 ]] && exit_status=${last_err} printf -- $exit_status | nxf_s3_upload - s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/43d1b8afbd14cc77e44b4760386d15/.exitcode || true set +u rm -rf $NXF_SCRATCH || true exit $exit_status } on_term() { set +e [[ "$pid" ]] && nxf_kill $pid } nxf_launch() { /bin/bash -Ceuo pipefail .command.run nxf_trace } nxf_stage() { true # stage input files downloads=(true) rm -f 1136_1VP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_6WU-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_5JZ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_UZP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_2RO-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_2E7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_5Z2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_42G-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_OMQ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1029_YCN-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_VTY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_7OO-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_VV1-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_3FA-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_I3-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_7L-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_0KC-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_3I9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_0L8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_4E9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_ED4-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_7BW-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_35B-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_03T-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_5PQ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_NTY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_77Q-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_CPH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_DNH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_P6B-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_42D-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f manifest.tsv rm -f .command.run rm -f 1136_65R-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1029_BTXK-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_3JH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_4DP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_6K6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_2PU-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_FH7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_5BE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1029_BNK6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_K8V-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_1VE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_TB0-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_7VJ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_KAF-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_D36-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_2QY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_HI8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_6F7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_OFT-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f .command.sh rm -f 1136_3FE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1029_PAP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_LSE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_814-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_3X6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_5AH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_DOI-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_0QH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_ROX-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_1I8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_3T4-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_2S9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_RWW-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1029_XI2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_75V-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1136_2F2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 1029_FSS-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_YP6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_XYF-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz rm -f 869_HV2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7e/4c8fec6a74a92b3c3770698d7562ab/1136_1VP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_1VP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7c/1a2d7fd681ab57c0bead03d03f33d7/1136_6WU-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_6WU-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/42/1e2a93321152b2926984c2b4994a18/1136_5JZ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_5JZ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/f9/8d6ed922ce93e4770f0c8bb229b7af/869_UZP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_UZP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/a8/102ad353b10394544040f0066e8a2b/1136_2RO-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_2RO-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/91/d1b526a8177f02d3db7b5ac3d7ece5/1136_2E7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_2E7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/89/869afd68be543239e3769c71bab69d/1136_5Z2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_5Z2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/c7/2dfa11f592c001f1b3d4f9f478abb7/1136_42G-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_42G-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/f4/3bf3b434ddccca3a71cb70584c17a6/869_OMQ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_OMQ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/36/f914192c2db19e4ec98c9209465592/1029_YCN-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1029_YCN-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/08/781a687600e4de1f4f7758661981e7/869_VTY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_VTY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/5e/8242856678e28e77b90ba712957269/1136_7OO-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_7OO-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7c/357814c5e318f4cee78227efa3473b/869_VV1-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_VV1-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/84/12b517c35cbe0e543a34282ce989e1/1136_3FA-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_3FA-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/6e/ac606e02b66e21523e29eb1e02ef7e/869_I3-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_I3-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/67/81422fd6322ac958783130caa4e245/1136_7L-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_7L-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/1f/7308f3d4ab5f3076988fac16d0ce9e/1136_0KC-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_0KC-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/4d/32686b29f22fb0ff7afa9a035d58cd/1136_3I9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_3I9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/12/3f5d3162ad1fd6c2117198eded5f7e/1136_0L8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_0L8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/ae/bef677b1d39335b751de4751a20215/1136_4E9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_4E9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/94/f9aa4675d3faa56ec6ce7902fb0a26/869_ED4-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_ED4-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/d5/976fee8938a56ac89e5dd5886495c0/1136_7BW-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_7BW-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/1a/2f3cac9611048bcc0059c2586fbddf/1136_35B-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_35B-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/4b/51e0e3c762fc9890ea8e59e3a7f0a8/1136_03T-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_03T-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/85/2df597875158095efe4fe39c04a9df/1136_5PQ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_5PQ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/d3/11ca954c43a8b43f20d8c55e98383e/869_NTY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_NTY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/2f/282803a95507ec571d4d0ee58cee08/1136_77Q-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_77Q-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/e4/8335bc4aa3eeacdd431db667c7c7f4/869_CPH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_CPH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/cd/de275e8400c1542af5d690c009bdaf/869_DNH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_DNH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/09/ff72030db2d4f4cedb845f84dd34e4/869_P6B-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_P6B-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/f5/02125d8a83ca33c11fd64bd4d1cda9/1136_42D-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_42D-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/tmp/7b/24e5d7b88ae196324c6460ea681857/manifest.tsv manifest.tsv") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/43d1b8afbd14cc77e44b4760386d15/.command.run .command.run") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/48/530381303e1406d4273543c8d75bbc/1136_65R-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_65R-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/10/1f19169ece8b476bc66a0f4a0204af/1029_BTXK-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1029_BTXK-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/b103d4f6f8d574fdd225274aad7e12/1136_3JH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_3JH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/cd/9708be8416725331ce2d17426e9759/1136_4DP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_4DP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/54/e95bf363b607465875c0187dc11a5f/1136_6K6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_6K6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/91/c26b94c3188423f71cf0f73d959a21/1136_2PU-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_2PU-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/b0/b9d7946eef04d420e9cdd2c9c7a907/869_FH7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_FH7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/94/c588f6c828fa417d256467386d7eb2/1136_5BE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_5BE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/a1/8f73daef092c12ad7d22934de629d4/1029_BNK6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1029_BNK6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/c0/bd4a2c918148e03df895f34870ed96/869_K8V-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_K8V-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/ab/e54a9797fc664d2f96a17aba102b6f/1136_1VE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_1VE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/3d/be8ce33c3a0dbf8b1f6b57defe2d64/869_TB0-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_TB0-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/a7/4b69a180ea2f6f548bad4e76819c48/1136_7VJ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_7VJ-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/f9/251f5de92613973c89e1ef2c6f1905/869_KAF-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_KAF-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/32/b949f1b11c281b2b04bfd70a313734/869_D36-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_D36-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/1f/93746988ebd62c0ece36ae3118f6d8/1136_2QY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_2QY-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/5c/3b1ef8bbd783c5bfedfafe46c72199/869_HI8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_HI8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/48/a8fe5b953d4959d735a2c22be8625a/1136_6F7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_6F7-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/66/50561745fbfc45a58dd9e883ea6af0/869_OFT-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_OFT-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/43d1b8afbd14cc77e44b4760386d15/.command.sh .command.sh") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/84/de2bf4fe6ab968b693beffe9ec65dd/1136_3FE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_3FE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/8a/f4abccdce972818f25356bfe8dd210/1029_PAP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1029_PAP-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/e8/172c112dea3e5ad928a916cf52c97e/869_LSE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_LSE-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/8a/eecb046b3e0781c3bb5144950820d6/1136_814-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_814-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/6f/00bfdf8b395366a877c8aa20970f87/1136_3X6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_3X6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/df/44b01b94b8158777c7f64e6b98ecc4/1136_5AH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_5AH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/2f/ecb362162eb89a88335633bbec3212/869_DOI-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_DOI-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/83/75973532b07d38e3b9d0a104edbb1d/1136_0QH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_0QH-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/9d/376ede54837d376c3878941e1000d2/869_ROX-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_ROX-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/5a/798e5ef8ac0310a00587583a5431f3/1136_1I8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_1I8-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/e5/6a067c19bee6c469a6399fc1da02b1/1136_3T4-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_3T4-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/24/ec31666a1ff3c6a426140c308f633d/1136_2S9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_2S9-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7f/912a3907449bfe71c590b90a77ade7/869_RWW-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_RWW-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/50/cba6901d9f33e9beb97fa2f8fa7cc2/1029_XI2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1029_XI2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/ed/e63f7e98e65682665f0ff372c8b53e/1136_75V-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_75V-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/b6/01ebc8364ad742fbb9dfbee7de7d29/1136_2F2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1136_2F2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/c1/3818d664ed5f6071997a4002eabad6/1029_FSS-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 1029_FSS-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/a1/46e639ce431479dd9571701ba3ba80/869_YP6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_YP6-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/4b/a70d1a36d3118903f928aab54948c0/869_XYF-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_XYF-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") downloads+=("nxf_s3_download s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/09/ac639e2810e989ea42e9706f8c630f/869_HV2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz 869_HV2-N1-BDNA-1_A23H3NYLT3_1_probes.tsv.gz") nxf_parallel "${downloads[@]}" echo "==> STAGING COMPLETE (73 inputs)" echo "" } nxf_unstage_outputs() { true uploads=() IFS=$'\n' for name in $(eval "ls -1d batch_reference.parquet sex_calls.csv feature_stats.csv versions.yml" | sort | uniq); do uploads+=("nxf_s3_upload '$name' s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/43d1b8afbd14cc77e44b4760386d15") done unset IFS nxf_parallel "${uploads[@]}" } nxf_unstage_controls() { true nxf_s3_upload .command.out s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/43d1b8afbd14cc77e44b4760386d15 || true nxf_s3_upload .command.err s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/43d1b8afbd14cc77e44b4760386d15 || true nxf_s3_upload .command.trace s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/43d1b8afbd14cc77e44b4760386d15 || true } nxf_unstage() { if [[ ${nxf_main_ret:=0} == 0 ]]; then (set -e -o pipefail; (nxf_unstage_outputs | tee -a .command.out) 3>&1 1>&2 2>&3 | tee -a .command.err) nxf_unstage_ret=$? fi nxf_unstage_controls } nxf_main() { trap on_exit EXIT trap on_term TERM INT USR2 trap '' USR1 [[ "${NXF_CHDIR:-}" ]] && cd "$NXF_CHDIR" NXF_SCRATCH="$(set +u; nxf_mktemp /tmp)" [[ $NXF_DEBUG > 0 ]] && nxf_env echo start | nxf_s3_upload - s3://natera-rnd-pltf-dev-nextflow-scratch-01/work/7a/43d1b8afbd14cc77e44b4760386d15/.command.begin set +u set -u [[ $NXF_SCRATCH ]] && cd $NXF_SCRATCH export NXF_TASK_WORKDIR="$PWD" nxf_stage set +e (set -o pipefail; (nxf_launch | tee .command.out) 3>&1 1>&2 2>&3 | tee .command.err) & pid=$! wait $pid || nxf_main_ret=$? nxf_unstage } $NXF_ENTRY