In [ ]:
Image('plots/cum_hist.png')
Out[ ]:
import os
import sys
import shutil
import glob
import matplotlib.pyplot as plt
from matplotlib import rc
rc('text', usetex = True)
plt.rcParams.update( # try to match font sizes of document
{'axes.labelsize': 20,
'axes.titlesize': 20,
'legend.fontsize': 20,
'xtick.labelsize': 20,
'ytick.labelsize': 20,
'text.usetex': True,
'font.family': 'serif',
'font.serif': ['palatino'],
'savefig.dpi': 300
})
%pylab inline
from glue.ligolw import ligolw
from glue.ligolw import table
from glue.ligolw import lsctables
from glue.ligolw import ilwd
from glue.ligolw import utils as ligolw_utils
import pycbc.strain
import pycbc.psd
from pycbc.pnutils import mass1_mass2_to_mchirp_eta
from pycbc.waveform import td_approximants, fd_approximants
from pycbc.waveform import get_two_pol_waveform_filter, get_td_waveform
from pycbc import DYN_RANGE_FAC
from pycbc.types import FrequencySeries, zeros
from pycbc.filter import match, overlap, sigma, make_frequency_series
from pycbc.scheme import CPUScheme, CUDAScheme
from pycbc import pnutils
from gwnr.Utils.SupportFunctions import make_padded_frequency_series
from gwnr.DataAnalysis import get_unique_hex_tag
import gwnr.DataAnalysis as DA
sys.path.append('/home/prayush.kumar/src/gwnrtools/bin/')
#sys.path.append('/home/prayush.kumar/local/venv/pycbc_master_enigma/src/gwnrtools/bin/')
Populating the interactive namespace from numpy and matplotlib
/home/prayush.kumar/local/venv/pycbc_master_enigma/lib/python2.7/site-packages/IPython/core/magics/pylab.py:161: UserWarning: pylab import has clobbered these variables: ['rc'] `%matplotlib` prevents importing * from pylab and numpy "\n`%matplotlib` prevents importing * from pylab and numpy"
Could not import ligolw in /home/prayush.kumar/src/GWNRTools/GWNRTools/Stats/FisherMatrixUtilities.pyc, LIGO XML tables wont be read
run_dir = '/home/prayush.kumar/projects/template_banks/testing_banksim'
try: os.makedirs(run_dir)
except: pass
os.chdir(run_dir)
!ls
!rm -fr *
rm: cannot remove ‘log’: Directory not empty
!cp ../seed_banks/non_spinning/NonSpin_m1_10_0_50_0_m2_10_0_50_0_MM_0_95.xml bank.xml
# !cp ../seed_banks/non_spinning/NonSpin_m1_5_0_50_0_m2_5_0_50_0_MM_0_95.xml bank.xml
!ls
bank gwnrtools_banksim_match_combine.sub banksim.dag gwnrtools_banksim_plots.sub banksim.dag.condor.sub gwnrtools_banksim.sub banksim.dag.dagman.log injection banksim.dag.dagman.out inj.xml banksim.dag.lib.err log banksim.dag.lib.out match banksim.dag.metrics match-part banksim.dag.nodes.log partial_results.sh banksim.dax plots banksim.ini results.dat banksim.sh scripts bank.xml submit.sh gwnrtools_banksim_collect_results.sub sub.out
def parse_match_file(mfile_name,
mvals_for_each_bank_point = {},
mvals_for_each_test_point = {}):
if not os.path.exists(mfile_name):
raise IOError("Provided file {} not found.".format(mfile_name))
with open(mfile_name, 'r') as mfile:
for line in mfile.readlines():
line = line.split()
btag, ptag = line[:2]
if btag not in mvals_for_each_bank_point:
mvals_for_each_bank_point[btag] = {}
if ptag not in mvals_for_each_bank_point[btag]:
mvals_for_each_bank_point[btag][ptag] = list([float(line[2]), float(line[3]), float(line[4])])
if ptag not in mvals_for_each_test_point:
mvals_for_each_test_point[ptag] = {}
if btag not in mvals_for_each_test_point[ptag]:
mvals_for_each_test_point[ptag][btag] = list([float(line[2]), float(line[3]), float(line[4])])
return mvals_for_each_bank_point, mvals_for_each_test_point
def get_all_matches_against_point(p, mvals_dict):
mkeys = mvals_dict[p].keys()
return np.array([mvals_dict[p][k] for k in mkeys], dtype=np.float128), mkeys
ALGORITHM
Outline:
bank.xml and a range of parameters + template model + signal model,bank.xml and each of the test injections,bank.xml,Simplification of algorithm for implementation:
gwnr_sample_parameter_space to generate the
requested injections. Store it as inj.xml.inj.xml into sub-parts of N_injection_per_job test points each. Store them
as injection/injection*.xml.bank.xml into sub-parts of N_template_per_job templates each.
Store them as bank/bank*.xml.injection/injectionA.xml and templates
bank/bankB.xml and store them as match-part/matchApartB.dat. This is done using
gwnr_banksim.gwnr_banksim_match_combine to read in all overlap data from
match-part/matchApart*.dat for injection/injectionA.xml; combine that information
by maximizing the match between each injection and all templates! Store the output
as match/matchA.dat.gwnr_banksim_collect_results to collect all match values from
match/match*.dat into a single file.Technical detail
match-part/matchApartB.dat?match/matchA.dat?!pwd
/home/prayush.kumar/projects/template_banks/testing_banksim
with open("banksim.ini", 'a') as fout:
fout.write("""\
[inspinj]
num-new-points = 5000
component-mass-min = 10.0
component-mass-max = 50.0
total-mass-max = 100.0
spin-component-min = 0
spin-component-max = 0
spin-mag-min = 0
spin-mag-max = 0
eccentricity-min = 0
eccentricity-max = 0
coa-phase-min = 0
coa-phase-max = 0
inclination-min = 0
inclination-max = 0
long-asc-nodes-min = 0
long-asc-nodes-max = 0
mean-per-ano-min = 0
mean-per-ano-max = 0
mchirp-window = 0
eccentricity-window = 0
verbose =
[executables]
inspinj = /home/prayush/src/gwnrtools/bin/gwnr_sample_parameter_space
banksim = /home/prayush/src/gwnrtools/bin/gwnr_banksim
[workflow]
accounting-group = ligo.dev.o3.cbc.explore.test
templates-per-job = 100
log-path = /usr1/prayush.kumar/
bank-file = /home/prayush.kumar/projects/template_banks/testing_banksim/bank.xml
injections-per-job = 10
templates-per-job = 10000
[banksim]
;processing-scheme = mkl
psd-model = aLIGOZeroDetHighPower
template-approximant = SEOBNRv4_ROM
proposal-approximant = SEOBNRv4_ROM
template-batch-size = 100
signal-batch-size = 100
low-frequency-cutoff = 20
sample-rate = 16384
signal-length = 64
mchirp-window = 0.10
verbose =
""")
!ls
banksim.ini bank.xml log
!rm bank banksim.da? banksim.sh inj.xml.xml log match injection banksim.dag* \
match-part partial_results.sh plots *.sub submit.sh scripts sub.out inj.xml -fr
!gwnr_create_banksim_workflow --config banksim.ini
2020-01-23 00:59:00,469 Making workspace directories 2020-01-23 00:59:00,546 Copying scripts 2020-01-23 00:59:00,558 Creating injection file Could not import ligolw in /home/prayush.kumar/src/GWNRTools/GWNRTools/Stats/FisherMatrixUtilities.pyc, LIGO XML tables wont be read 2020-01-23 00:59:05,156 | INFO : mchirp-window = 0.000000 2020-01-23 00:59:05,156 | INFO : eccentricity-window = 0.000000 2020-01-23 00:59:05,156 | INFO : Storing the new sample points in inj.xml 2020-01-23 00:59:05,165 | INFO : 0 points chosen 2020-01-23 00:59:05,183 | INFO : 100 points chosen 2020-01-23 00:59:05,207 | INFO : 200 points chosen 2020-01-23 00:59:05,239 | INFO : 300 points chosen 2020-01-23 00:59:05,277 | INFO : 400 points chosen 2020-01-23 00:59:05,323 | INFO : 500 points chosen 2020-01-23 00:59:05,377 | INFO : 600 points chosen 2020-01-23 00:59:05,439 | INFO : 700 points chosen 2020-01-23 00:59:05,512 | INFO : 800 points chosen 2020-01-23 00:59:05,592 | INFO : 900 points chosen 2020-01-23 00:59:05,681 | INFO : 1000 points chosen 2020-01-23 00:59:05,777 | INFO : 1100 points chosen 2020-01-23 00:59:05,878 | INFO : 1200 points chosen 2020-01-23 00:59:05,990 | INFO : 1300 points chosen 2020-01-23 00:59:06,114 | INFO : 1400 points chosen 2020-01-23 00:59:06,245 | INFO : 1500 points chosen 2020-01-23 00:59:06,378 | INFO : 1600 points chosen 2020-01-23 00:59:06,528 | INFO : 1700 points chosen 2020-01-23 00:59:06,678 | INFO : 1800 points chosen 2020-01-23 00:59:06,840 | INFO : 1900 points chosen 2020-01-23 00:59:07,004 | INFO : 2000 points chosen 2020-01-23 00:59:07,183 | INFO : 2100 points chosen 2020-01-23 00:59:07,376 | INFO : 2200 points chosen 2020-01-23 00:59:07,569 | INFO : 2300 points chosen 2020-01-23 00:59:07,782 | INFO : 2400 points chosen 2020-01-23 00:59:07,990 | INFO : 2500 points chosen 2020-01-23 00:59:08,218 | INFO : 2600 points chosen 2020-01-23 00:59:08,448 | INFO : 2700 points chosen 2020-01-23 00:59:08,685 | INFO : 2800 points chosen 2020-01-23 00:59:08,954 | INFO : 2900 points chosen 2020-01-23 00:59:09,213 | INFO : 3000 points chosen 2020-01-23 00:59:09,477 | INFO : 3100 points chosen 2020-01-23 00:59:09,741 | INFO : 3200 points chosen 2020-01-23 00:59:10,010 | INFO : 3300 points chosen 2020-01-23 00:59:10,295 | INFO : 3400 points chosen 2020-01-23 00:59:10,589 | INFO : 3500 points chosen 2020-01-23 00:59:10,882 | INFO : 3600 points chosen 2020-01-23 00:59:11,195 | INFO : 3700 points chosen 2020-01-23 00:59:11,512 | INFO : 3800 points chosen 2020-01-23 00:59:11,849 | INFO : 3900 points chosen 2020-01-23 00:59:12,184 | INFO : 4000 points chosen 2020-01-23 00:59:12,527 | INFO : 4100 points chosen 2020-01-23 00:59:12,896 | INFO : 4200 points chosen 2020-01-23 00:59:13,275 | INFO : 4300 points chosen 2020-01-23 00:59:13,660 | INFO : 4400 points chosen 2020-01-23 00:59:14,050 | INFO : 4500 points chosen 2020-01-23 00:59:14,459 | INFO : 4600 points chosen 2020-01-23 00:59:14,861 | INFO : 4700 points chosen 2020-01-23 00:59:15,275 | INFO : 4800 points chosen 2020-01-23 00:59:15,691 | INFO : 4900 points chosen 2020-01-23 00:59:16,114 | INFO : Writing 5000 new points to inj.xml 2020-01-23 00:59:16,447 Splitting template bank 2020-01-23 00:59:18,715 Splitting injection file 2020-01-23 00:59:35,794 Creating DAG 2020-01-23 00:59:54,055 DO : 500 SKIP 0 2020-01-23 00:59:54,085 Creating submit script 2020-01-23 00:59:54,778 Done
!ls && cat -n submit.sh
bank injection
banksim.dag inj.xml
banksim.dax log
banksim.ini match
banksim.sh match-part
bank.xml partial_results.sh
gwnrtools_banksim_collect_results.sub plots
gwnrtools_banksim_match_combine.sub scripts
gwnrtools_banksim_plots.sub submit.sh
gwnrtools_banksim.sub
1 #!/bin/bash
2 condor_submit_dag banksim.dag
3
!./submit.sh >> sub.out
!cat sub.out
Submitting job(s). 1 job(s) submitted to cluster 36348018. ----------------------------------------------------------------------- File for submitting this DAG to HTCondor : banksim.dag.condor.sub Log of DAGMan debugging messages : banksim.dag.dagman.out Log of HTCondor library output : banksim.dag.lib.out Log of HTCondor library error messages : banksim.dag.lib.err Log of the life of condor_dagman itself : banksim.dag.dagman.log -----------------------------------------------------------------------
!condor_rm 36347882
Couldn't find/remove all jobs in cluster 36347882
!tail -n10 banksim.dag.dagman.out
01/23/20 01:10:41 Of 1002 nodes total: 01/23/20 01:10:41 Done Pre Queued Post Ready Un-Ready Failed 01/23/20 01:10:41 === === === === === === === 01/23/20 01:10:41 1002 0 0 0 0 0 0 01/23/20 01:10:41 0 job proc(s) currently held 01/23/20 01:10:41 DAGMan Runtime Statistics: [ EventCycleTimeStd = 1.099006578522996; EventCycleTimeMax = 5.685035943984985; SubmitCycleTimeMin = 1.001358032226562E-05; SubmitCycleTimeAvg = 0.5562732588161122; SleepCycleTimeCount = 109.0; SubmitCycleTimeCount = 110.0; SubmitCycleTimeMax = 5.677910089492798; SleepCycleTimeMin = 5.000730991363525; SubmitCycleTimeSum = 61.19005846977234; LogProcessCycleTimeStd = 0.004210175588373334; SubmitCycleTimeStd = 1.093274638684228; LogProcessCycleTimeAvg = 0.00445103323137438; EventCycleTimeMin = 0.0001339912414550781; LogProcessCycleTimeMax = 0.02737689018249512; LogProcessCycleTimeMin = 6.890296936035156E-05; LogProcessCycleTimeSum = 0.3293764591217041; LogProcessCycleTimeCount = 74.0; EventCycleTimeSum = 61.58794665336609; SleepCycleTimeSum = 545.47145819664; SleepCycleTimeAvg = 5.00432530455633; SleepCycleTimeStd = 0.001371025622407103; SleepCycleTimeMax = 5.007020235061646; EventCycleTimeCount = 109.0; EventCycleTimeAvg = 0.5650270335171201; ] 01/23/20 01:10:41 Wrote metrics file banksim.dag.metrics. 01/23/20 01:10:41 Metrics not sent because of PEGASUS_METRICS or CONDOR_DEVELOPERS setting. 01/23/20 01:10:41 DAGMan Runtime Statistics: [ EventCycleTimeStd = 1.099006578522996; EventCycleTimeMax = 5.685035943984985; SubmitCycleTimeMin = 1.001358032226562E-05; SubmitCycleTimeAvg = 0.5562732588161122; SleepCycleTimeCount = 109.0; SubmitCycleTimeCount = 110.0; SubmitCycleTimeMax = 5.677910089492798; SleepCycleTimeMin = 5.000730991363525; SubmitCycleTimeSum = 61.19005846977234; LogProcessCycleTimeStd = 0.004210175588373334; SubmitCycleTimeStd = 1.093274638684228; LogProcessCycleTimeAvg = 0.00445103323137438; EventCycleTimeMin = 0.0001339912414550781; LogProcessCycleTimeMax = 0.02737689018249512; LogProcessCycleTimeMin = 6.890296936035156E-05; LogProcessCycleTimeSum = 0.3293764591217041; LogProcessCycleTimeCount = 74.0; EventCycleTimeSum = 61.58794665336609; SleepCycleTimeSum = 545.47145819664; SleepCycleTimeAvg = 5.00432530455633; SleepCycleTimeStd = 0.001371025622407103; SleepCycleTimeMax = 5.007020235061646; EventCycleTimeCount = 109.0; EventCycleTimeAvg = 0.5650270335171201; ] 01/23/20 01:10:41 **** condor_scheduniv_exec.36348018.0 (condor_DAGMAN) pid 1192333 EXITING WITH STATUS 0
print('''USE WITH CARE ... ''')
# !gwnr_force_success_from_submit_file gwnr_banksim.sub
USE WITH CARE ...
from IPython.display import Image
!ls plots
cum_hist.png m1-m2-mchirpdiff.png q-s2z-match90.png hist-match-cum.png m1-m2-mchirpreldiff.png q-s2z-match97.png hist-match.png q-s1m-match.png q-s2z-match.png hist-mchirp-diff.png q-s1z-match.png q-s2z-mchirpdiff.png hist-mchirp-reldiff.png q-s1z-mchirpdiff.png q-s2z-mchirpreldiff.png inc-match.png q-s1z-mchirpreldiff.png tm1-tm2-match.png m1-m2-match.png q-s2m-match.png
Image('plots/cum_hist.png')
Image('plots/m1-m2-match.png')
Image('plots/m1-m2-mchirpreldiff.png')
