Skip to content

Instantly share code, notes, and snippets.

@clelange

clelange/bsm-search-htc.yaml

Created July 29, 2020 20:53
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save clelange/c2da7cfc7de3e9a1194a92d0fb105e29 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save clelange/c2da7cfc7de3e9a1194a92d0fb105e29 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
bsm-search-htc.yaml
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
generateName: bsm-search-htc-
labels:
workflows.argoproj.io/archive-strategy: false
spec:
volumeClaimTemplates:
- metadata:
name: vol-cephfs-bsm
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10Gi
storageClassName: csi-cephfs-storageclass
arguments:
parameters:
- name: base-dir # a list of items
value: "/data/bsm-search-htc/"
entrypoint: databkgmc
templates:
- name: databkgmc
inputs:
parameters:
- name: base-dir
dag:
tasks:
- name: prepare-dir
template: cleanup-recreate-dir
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: all-bkg-mc
dependencies: [prepare-dir]
template: wflow-all-mc
arguments:
parameters:
- name: type
value: "{{item.type}}"
- name: mcweight
value: "{{item.mcweight}}"
- name: nevents
value: "{{item.nevents}}"
- name: njobs
value: "{{item.njobs}}"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
withItems:
- { type: 'mc1', mcweight: '0.01875', nevents: '40000', njobs: 4 }
- { type: 'mc2', mcweight: '0.0125', nevents: '40000', njobs: 4 }
- name: signal
dependencies: [prepare-dir]
template: workflow-signal
arguments:
parameters:
- name: type
value: "{{item.type}}"
- name: mcweight
value: "{{item.mcweight}}"
- name: nevents
value: "{{item.nevents}}"
- name: njobs
value: "{{item.njobs}}"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
withItems:
- { type: 'sig', mcweight: '0.0025', nevents: '40000', njobs: 2 }
- name: data
dependencies: [prepare-dir]
template: workflow-data
arguments:
parameters:
- name: type
value: "{{item.type}}"
- name: nevents
value: "{{item.nevents}}"
- name: njobs
value: "{{item.njobs}}"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
withItems:
- { type: 'data', nevents: '20000', njobs: 5 }
- name: merge-all
dependencies: [all-bkg-mc, signal, data]
template: merge-explicit
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfiles
value: "{{item.inputfiles}}"
- name: outputfile
value: "all_merged_hist.root"
withItems:
- { inputfiles: "mc1_merged_hist.root mc2_merged_hist.root sig_merged_hist.root data_merged_hist.root" }
- name: makews
dependencies: [merge-all]
template: makews-action
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: data_bkg_hists
value: "all_merged_hist.root"
- name: workspace_prefix
value: "{{inputs.parameters.base-dir}}/results"
- name: xml_dir
value: "{{inputs.parameters.base-dir}}/xmldir"
- name: plot
dependencies: [makews]
template: plot-action
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: combined_model
value: "{{inputs.parameters.base-dir}}/results_combined_meas_model.root"
- name: nominal_vals
value: "{{inputs.parameters.base-dir}}/nominal_vals.yml"
- name: fit_results
value: "{{inputs.parameters.base-dir}}/fit_results.yml"
- name: prefit_plot
value: "{{inputs.parameters.base-dir}}/prefit.pdf"
- name: postfit_plot
value: "{{inputs.parameters.base-dir}}/postfit.pdf"
- name: wflow-all-mc
inputs:
parameters:
- name: type
- name: mcweight
- name: nevents
- name: njobs
- name: base-dir
dag:
tasks:
- name: run-mc-batch
template: generate-batch
arguments:
parameters:
- name: type
value: "{{inputs.parameters.type}}"
- name: nevents
value: "{{inputs.parameters.nevents}}"
- name: njobs
value: "{{inputs.parameters.njobs}}"
- name: merge-mc-batch
dependencies: [run-mc-batch]
template: merge-root-batch
arguments:
parameters:
- name: base-dir-batch
value: "/data/htc_out/generate-batch-{{inputs.parameters.type}}/"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: type
value: "{{inputs.parameters.type}}"
- name: njobs
value: "{{inputs.parameters.njobs}}"
- name: select-mc
dependencies: [merge-mc-batch]
template: select
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}.root"
- name: suffix
value: "{{item.suffix}}"
- name: outputfile
value: "{{inputs.parameters.type}}_{{item.suffix}}.root"
- name: region
value: "{{item.region}}"
- name: variation
value: "{{item.variation}}"
withItems:
- { region: signal, variation: shape_conv_up, suffix: shape_conv_up }
- { region: signal, variation: shape_conv_dn, suffix: shape_conv_dn }
- { region: signal, variation: "nominal,weight_var1_up,weight_var1_dn", suffix: nominal }
- name: hist-shape
dependencies: [select-mc]
template: histogram-shape
arguments:
parameters:
- name: weight
value: "{{inputs.parameters.mcweight}}"
- name: variations # nominal
value: nominal
- name: shapevar # shape_conv_up shape_conv_dn
value: "{{item.shapevar}}"
- name: type
value: "{{inputs.parameters.type}}"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}_{{item.shapevar}}.root"
- name: outputfile
value: "{{inputs.parameters.type}}_{{item.shapevar}}_hist.root"
withItems:
- { shapevar: shape_conv_up }
- { shapevar: shape_conv_dn }
- name: hist-weight
dependencies: [select-mc]
template: histogram-weight
arguments:
parameters:
- name: weight
value: "{{inputs.parameters.mcweight}}"
- name: variations
value: "nominal,weight_var1_up,weight_var1_dn"
- name: shapevar # nominal
value: "{{item.shapevar}}"
- name: type
value: "{{inputs.parameters.type}}"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}_{{item.shapevar}}.root"
- name: outputfile
value: "{{inputs.parameters.type}}_{{item.shapevar}}_hist.root"
withItems:
- { shapevar: nominal }
- name: merge-hist-shape
dependencies: [hist-shape]
template: merge-explicit
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfiles
value: "{{item.inputfiles}}"
- name: outputfile
value: "{{inputs.parameters.type}}_shape_hist.root"
withItems:
- { inputfiles: "{{inputs.parameters.type}}_shape_conv_up_hist.root {{inputs.parameters.type}}_shape_conv_dn_hist.root" }
- name: merge-hist-all
dependencies: [merge-hist-shape,hist-weight]
template: merge-explicit
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfiles
value: "{{item.inputfiles}}"
- name: outputfile
value: "{{inputs.parameters.type}}_merged_hist.root"
withItems:
- { inputfiles: "{{inputs.parameters.type}}_nominal_hist.root {{inputs.parameters.type}}_shape_hist.root" }
- name: workflow-signal
inputs:
parameters:
- name: type
- name: nevents
- name: mcweight
- name: njobs
- name: base-dir
dag:
tasks:
- name: run-mc-batch
template: generate-batch
arguments:
parameters:
- name: type
value: "{{inputs.parameters.type}}"
- name: nevents
value: "{{inputs.parameters.nevents}}"
- name: njobs
value: "{{inputs.parameters.njobs}}"
- name: merge-mc-batch
dependencies: [run-mc-batch]
template: merge-root-batch
arguments:
parameters:
- name: base-dir-batch
value: "/data/htc_out/generate-batch-{{inputs.parameters.type}}/"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: type
value: "{{inputs.parameters.type}}"
- name: njobs
value: "{{inputs.parameters.njobs}}"
- name: select-mc
dependencies: [merge-mc-batch]
template: select
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}.root"
- name: suffix
value: "{{item.suffix}}"
- name: outputfile
value: "{{inputs.parameters.type}}_{{item.suffix}}.root"
- name: region
value: "{{item.region}}"
- name: variation
value: "{{item.variation}}"
withItems:
- { region: signal, variation: nominal, suffix: nominal }
- name: hist-weight
dependencies: [select-mc]
template: histogram-weight
arguments:
parameters:
- name: weight
value: "{{inputs.parameters.mcweight}}"
- name: variations
value: "nominal"
- name: shapevar # nominal
value: "{{item.shapevar}}"
- name: type
value: "{{inputs.parameters.type}}nal"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}_{{item.shapevar}}.root"
- name: outputfile
value: "{{inputs.parameters.type}}_{{item.shapevar}}_hist.root"
withItems:
- { shapevar: nominal }
- name: merge-hist-all
dependencies: [hist-weight]
template: merge-explicit
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfiles
value: "{{item.inputfiles}}"
- name: outputfile
value: "{{inputs.parameters.type}}_merged_hist.root"
withItems:
- { inputfiles: "{{inputs.parameters.type}}_nominal_hist.root" }
- name: workflow-data
inputs:
parameters:
- name: type
- name: nevents
- name: njobs
- name: base-dir
dag:
tasks:
- name: run-data-batch
template: generate-batch
arguments:
parameters:
- name: type
value: "{{inputs.parameters.type}}"
- name: nevents
value: "{{inputs.parameters.nevents}}"
- name: njobs
value: "{{inputs.parameters.njobs}}"
- name: merge-data-batch
dependencies: [run-data-batch]
template: merge-root-batch
arguments:
parameters:
- name: base-dir-batch
value: "/data/htc_out/generate-batch-{{inputs.parameters.type}}/"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: type
value: "{{inputs.parameters.type}}"
- name: njobs
value: "{{inputs.parameters.njobs}}"
- name: select-signal
dependencies: [merge-data-batch]
template: select
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}.root"
- name: suffix
value: "{{item.suffix}}"
- name: outputfile
value: "{{inputs.parameters.type}}_{{item.suffix}}.root"
- name: region
value: "{{item.region}}"
- name: variation
value: "{{item.variation}}"
withItems:
- { region: signal, variation: nominal, suffix: signal }
- name: select-control
dependencies: [merge-data-batch]
template: select
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}.root"
- name: suffix
value: "{{item.suffix}}"
- name: outputfile
value: "{{inputs.parameters.type}}_{{item.suffix}}.root"
- name: region
value: "{{item.region}}"
- name: variation
value: "{{item.variation}}"
withItems:
- { region: control, variation: nominal, suffix: control }
- name: hist-signal
dependencies: [select-signal]
template: histogram-weight
arguments:
parameters:
- name: weight
value: 1.0
- name: variations
value: "nominal"
- name: shapevar # nominal
value: "{{item.shapevar}}"
- name: type
value: "{{inputs.parameters.type}}"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}_signal.root"
- name: outputfile
value: "{{inputs.parameters.type}}_hist.root"
withItems:
- { shapevar: nominal }
- name: hist-control
dependencies: [select-control]
template: histogram-weight
arguments:
parameters:
- name: weight
value: 0.1875
- name: variations
value: "nominal"
- name: shapevar # nominal
value: "{{item.shapevar}}"
- name: type
value: "{{item.type}}"
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfile
value: "{{inputs.parameters.type}}_control.root"
- name: outputfile
value: "{{item.type}}_hist.root"
withItems:
- { shapevar: nominal, type: qcd }
- name: merge-hist-all
dependencies: [hist-signal, hist-control]
template: merge-explicit
arguments:
parameters:
- name: base-dir
value: "{{inputs.parameters.base-dir}}"
- name: inputfiles
value: "{{item.inputfiles}}"
- name: outputfile
value: "{{inputs.parameters.type}}_merged_hist.root"
withItems:
- { inputfiles: "{{inputs.parameters.type}}_hist.root qcd_hist.root" }
- name: cleanup-recreate-dir
inputs:
parameters:
- name: base-dir
script:
image: reanahub/reana-demo-bsm-search
command: [bash]
source: |
set -x
rm -rf {{inputs.parameters.base-dir}}
rm -rf /data/htc_out/generate-batch-mc1
rm -rf /data/htc_out/generate-batch-mc2
rm -rf /data/htc_out/generate-batch-sig
rm -rf /data/htc_out/generate-batch-data
mkdir -p {{inputs.parameters.base-dir}}
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
- name: scatter-template
inputs:
parameters:
- name: njobs
script:
image: python:alpine3.6
command: [python]
source: |
import json
import sys
json.dump([i for i in range({{inputs.parameters.njobs}})], sys.stdout)
- name: generate
inputs:
parameters:
- name: type
- name: nevents
- name: jobnumber
- name: base-dir
script:
image: reanahub/reana-demo-bsm-search
command: [bash]
source: |
set -x
source /usr/local/bin/thisroot.sh
python /code/generantuple.py {{inputs.parameters.type}} {{inputs.parameters.nevents}} {{inputs.parameters.base-dir}}/{{inputs.parameters.type}}_{{inputs.parameters.jobnumber}}.root
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
# outputs:
# artifacts:
# # generate hello-art artifact from /tmp/hello_world.txt
# # artifacts can be directories as well as files
# - name: outputfile
# path: /tmp/{{inputs.parameters.outputfile}}
- name: generate-batch
inputs:
parameters:
- name: type
- name: nevents
- name: njobs
resource:
action: create
successCondition: status.succeeded == {{inputs.parameters.njobs}}
failureCondition: status.failed > 0
manifest: |
apiVersion: htc.cern.ch/v1alpha1
kind: HTCJob
metadata:
name: generate-batch-{{inputs.parameters.type}}
generateName: htcj-generate-batch--{{inputs.parameters.type}}-
ownerReferences:
- apiVersion: argoproj.io/v1alpha1
blockOwnerDeletion: true
kind: Workflow
name: "{{workflow.name}}"
uid: "{{workflow.uid}}"
spec:
name: generate-batch-{{inputs.parameters.type}}
queue: {{inputs.parameters.njobs}}
script:
image: reanahub/reana-demo-bsm-search
command: bash
source: |
env
source /usr/local/bin/thisroot.sh
python /code/generantuple.py {{inputs.parameters.type}} {{inputs.parameters.nevents}} /htc_run/{{inputs.parameters.type}}_${HTCPROCID}.root
ls -l
- name: select
inputs:
parameters:
- name: inputfile
- name: outputfile
- name: region
- name: base-dir
- name: variation
script:
image: reanahub/reana-demo-bsm-search
command: [bash]
source: |
set -x
BASE_DIR={{inputs.parameters.base-dir}}
source /usr/local/bin/thisroot.sh
python /code/select.py ${BASE_DIR}{{inputs.parameters.inputfile}} ${BASE_DIR}{{inputs.parameters.outputfile}} {{inputs.parameters.region}} {{inputs.parameters.variation}}
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
- name: merge-root
inputs:
parameters:
- name: type
- name: njobs
- name: base-dir
script:
image: reanahub/reana-env-root6:6.18.04
command: [bash]
source: |
set -x
BASE_DIR={{inputs.parameters.base-dir}}
BASE={{inputs.parameters.type}}
END={{inputs.parameters.njobs}}
INPUTS=""
for (( c=0; c<$END; c++ )); do
INPUTS="$INPUTS $(printf ${BASE_DIR}/${BASE}_${c}.root)"
done
source /usr/local/bin/thisroot.sh
hadd -f ${BASE_DIR}/${BASE}.root ${INPUTS}
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
- name: merge-root-batch
inputs:
parameters:
- name: type
- name: njobs
- name: base-dir-batch
- name: base-dir
script:
image: reanahub/reana-env-root6:6.18.04
command: [bash]
source: |
set -x
BASE_DIR={{inputs.parameters.base-dir}}
BASE_DIR_BATCH={{inputs.parameters.base-dir-batch}}
BASE={{inputs.parameters.type}}
END={{inputs.parameters.njobs}}
INPUTS=""
for (( c=0; c<$END; c++ )); do
INPUTS="$INPUTS $(printf ${BASE_DIR_BATCH}/${BASE}_${c}.root)"
done
source /usr/local/bin/thisroot.sh
hadd -f ${BASE_DIR}/${BASE}.root ${INPUTS}
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
- name: merge-explicit
inputs:
parameters:
- name: outputfile
- name: inputfiles
- name: base-dir
script:
image: reanahub/reana-env-root6:6.18.04
command: [bash]
source: |
set -x
BASE_DIR={{inputs.parameters.base-dir}}
inputfiles="{{inputs.parameters.inputfiles}}"
echo $inputfiles
INPUTS=""
for i in $inputfiles; do
INPUTS="$INPUTS $(printf $BASE_DIR/$i)"
done
source /usr/local/bin/thisroot.sh
hadd -f ${BASE_DIR}{{inputs.parameters.outputfile}} ${INPUTS}
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
- name: histogram-weight
inputs:
parameters:
- name: outputfile
- name: inputfile
- name: base-dir
- name: weight
- name: variations # nominal,weight_var1_up,weight_var1_dn
- name: shapevar # nominal
- name: type
script:
image: reanahub/reana-demo-bsm-search
command: [bash]
source: |
set -x
BASE_DIR={{inputs.parameters.base-dir}}
source /usr/local/bin/thisroot.sh
variations=$(echo {{inputs.parameters.variations}}|sed 's| |,|g')
name="{{inputs.parameters.type}}"
python /code/histogram.py ${BASE_DIR}{{inputs.parameters.inputfile}} ${BASE_DIR}{{inputs.parameters.outputfile}} $name {{inputs.parameters.weight}} $variations
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
- name: histogram-shape
inputs:
parameters:
- name: outputfile
- name: inputfile
- name: base-dir
- name: weight
- name: variations # nominal
- name: shapevar # shape_conv_up shape_conv_dn
- name: type
script:
image: reanahub/reana-demo-bsm-search
command: [bash]
source: |
set -x
BASE_DIR={{inputs.parameters.base-dir}}
source /usr/local/bin/thisroot.sh
variations=$(echo {{inputs.parameters.variations}}|sed 's| |,|g')
name="{{inputs.parameters.type}}_{{inputs.parameters.shapevar}}"
python /code/histogram.py ${BASE_DIR}{{inputs.parameters.inputfile}} ${BASE_DIR}{{inputs.parameters.outputfile}} {{inputs.parameters.type}} {{inputs.parameters.weight}} $variations $name
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
- name: makews-action
inputs:
parameters:
- name: base-dir
- name: xml_dir
- name: workspace_prefix
- name: data_bkg_hists
script:
image: reanahub/reana-demo-bsm-search
command: [bash]
source: |
set -x
source /usr/local/bin/thisroot.sh
python /code/makews.py {{inputs.parameters.base-dir}}{{inputs.parameters.data_bkg_hists}} {{inputs.parameters.workspace_prefix}} {{inputs.parameters.xml_dir}}
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
- name: plot-action
inputs:
parameters:
- name: base-dir
- name: combined_model
- name: nominal_vals
- name: fit_results
- name: prefit_plot
- name: postfit_plot
script:
image: reanahub/reana-demo-bsm-search
command: [bash]
source: |
set -x
source /usr/local/bin/thisroot.sh
hfquickplot write-vardef {{inputs.parameters.combined_model}} combined {{inputs.parameters.nominal_vals}}
hfquickplot plot-channel {{inputs.parameters.combined_model}} combined channel1 x {{inputs.parameters.nominal_vals}} -c qcd,mc2,mc1,signal -o {{inputs.parameters.prefit_plot}}
hfquickplot fit {{inputs.parameters.combined_model}} combined {{inputs.parameters.fit_results}}
hfquickplot plot-channel {{inputs.parameters.combined_model}} combined channel1 x {{inputs.parameters.fit_results}} -c qcd,mc2,mc1,signal -o {{inputs.parameters.postfit_plot}}
volumeMounts:
- mountPath: /data/
name: vol-cephfs-bsm
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment