#!/usr/bin/env python
import os
import sys
import time
import datetime
import re
import shutil
import shlex
from optparse import OptionParser
try:
    import xml.etree.cElementTree as etree
except ImportError:
    import xml.etree.ElementTree as etree

# Use pegasus-config to find our lib path
home_dir = os.path.normpath(os.path.join(os.path.dirname(sys.argv[0]), "..", ".."))
python_dir = os.path.join(home_dir, "lib", "pegasus", "python")

# also need the python externals dir
os.sys.path.insert(0, python_dir)

from Pegasus.tools import utils

class DAG:
    def __init__(self):
        self.jobs = {}
        self.edges = []

class DAX:
    def __init__(self):
        self.tasks = {}
        self.edges = []

class Job: pass

class Task: pass

class File:
    def __init__(self):
        self.name = None
        self.link = None
        self.optional = None
        self.type = None
        self.size = None
        self.bread = None
        self.bwrite = None

class Invocation:
    def __init__(self):
        self.duration = None
        self.xform = None
        self.derivation = None
        self.site = None
        self.host = None
        self.start = None
        self.cpu = None
        self.cores = None
        self.sent = None
        self.received = None
        self.procs = []
        self.files = []
        self.arguments = []
        self.connections = []
        self.slot = None

class Braindump:
    def __init__(self, bdfile):
        self.filename = bdfile
        self.submitdir = os.path.abspath(os.path.dirname(bdfile))
        self.entries = {}
        self._parse()

    def _parse(self):
        f = open(self.filename, "r")
        try:
            self.entries = dict([l.strip().split(None, 1) for l in f])
        finally:
            f.close()

    def get_path(self, name):
        path = self[name]
        if path is None:
            return None

        # If the file exists, then return it
        if os.path.isfile(path):
            return path

        # Otherwise, try to see if the path to the submitdir has changed
        # and try to reconstruct the path
        filename = os.path.basename(path)
        newpath = os.path.join(self.submitdir, filename)

        if os.path.isfile(newpath):
            return newpath

        return path

    def __getitem__(self, name):
        if name not in self.entries:
            return None
        return self.entries[name]

    def __repr__(self):
        return repr(self.entries)

def extract_arguments(job):
    elem = job.find("./argument")

    if elem is None:
        return []

    args = []

    if elem.text is not None:
        args.append(elem.text)

    for f in elem:
        if f.text is not None:
            args.append(f.text)

        if "name" in f.attrib:
            args.append(f.attrib["name"])

        if "file" in f.attrib:
            args.append(f.attrib["file"])

        if f.tail is not None:
            args.append(f.tail)

    if elem.tail is not None:
        args.append(elem.tail)

    # Fix the whitespace
    args = shlex.split("".join(args))

    return args

def extract_files(job):
    files = []

    for e in job:
        if e.tag == "uses":
            f = File()
            if "file" in e.attrib:
                f.name = e.attrib["file"]
            else:
                f.name = e.attrib["name"]
            f.link = e.attrib["link"]
            if "optional" in e.attrib:
                f.optional = e.attrib["optional"].lower() == 'true'
            else:
                f.optional = False
            if "type" in e.attrib:
                f.type = e.attrib["type"]
            else:
                f.type = "data"
            files.append(f)

    return files

def parse_dax(daxfile):
    print "Parsing DAX..."

    dax = DAX()

    root = etree.parse(daxfile).getroot()

    dax.name = root.attrib["name"]

    # Strip all the damned namespaces
    for e in root.findall(".//"):
        ignore, e.tag = e.tag[1:].rsplit("}", 1)

    for e in root:
        if e.tag == "job":
            task = Task()
            task.id = e.attrib["id"]
            task.xform = e.attrib["name"]
            if "namespace" in e.attrib:
                task.xform = e.attrib["namespace"] + "::" + task.xform
            if "version" in e.attrib:
                task.xform = task.xform + ":" + e.attrib["version"]
            if "dv-name" in e.attrib:
                task.derivation = e.attrib["dv-name"]
                if "dv-version" in e.attrib:
                    task.derivation += ":" + e.attrib["dv-version"]
            else:
                task.derivation = task.id

            task.arguments = extract_arguments(e)

            task.files = extract_files(e)

            dax.tasks[task.id] = task
        elif e.tag == "child":
            child = dax.tasks[e.attrib["ref"]]
            for p in e:
                parent = dax.tasks[p.attrib["ref"]]
                dax.edges.append((parent,child))
        elif e.tag in ["file","filename","executable","invoke"]:
            pass
        else:
            print "UNEXPECTED TAG: %s" % e.tag
            exit(1)

    print "%d tasks and %d edges" % (len(dax.tasks), len(dax.edges))

    return dax

def parse_dag(dagfile):
    print "Parsing DAG..."

    dag = DAG()
    dag.name = os.path.basename(dagfile)

    dag.dir = os.path.dirname(dagfile)

    f = open(dagfile, "r")

    for l in f:
        l = l.strip()
        if len(l) == 0 or l.startswith("#"):
            continue

        if l.startswith("JOB"):
            rec = l.split()
            job = Job()
            job.name = rec[1]
            dag.jobs[job.name] = job
            submit_file = os.path.join(dag.dir, job.name + ".sub")
            parse_submit_file(submit_file, job)
        elif l.startswith("PARENT"):
            rec = l.split()
            parent_name = rec[1]
            child_name = rec[3]
            parent = dag.jobs[parent_name]
            child = dag.jobs[child_name]
            dag.edges.append((parent,child))

    print "%d jobs and %d edges" % (len(dag.jobs), len(dag.edges))

    return dag

def parse_submit_file(submit_file, job):
    if not os.path.isfile(submit_file):
        raise Exception("Submit file not found: %s" % submit_file)

    f = open(submit_file)
    try:
        for l in f:
            if l.lstrip().startswith("priority"):
                job.priority = l.split("=")[1].strip()
    finally:
        f.close()

def splitrecord(line):
    rec = line.split()
    jobid = rec[1].strip("()")
    ts = rec[2] + '/2009 ' + rec[3]
    return jobid, ts

def parse_job_log(logfile, dag):
    print "Parsing job log..."

    f = open(logfile, "r")

    condor = {}

    for l in f:
        l = l.strip()

        if len(l) == 0 or l.startswith("#"):
            continue

        if l.startswith("000"):
            # submit
            jobid, ts = splitrecord(l)
            while not l.startswith("..."):
                l = f.next().strip()
                if l.startswith("DAG Node"):
                    rec = l.split(":")
                    name = rec[1].strip()
            job = dag.jobs[name]
            job.id = jobid
            job.submit = ts

            # XXX Old local universe jobs did not have an 001 event
            # so we have to set the execute timestamp = submit ts
            job.execute = ts

            condor[jobid] = job
        elif l.startswith("001"):
            # execute
            jobid, ts = splitrecord(l)
            condor[jobid].execute = ts
        elif l.startswith("005"):
            # terminate
            jobid, ts = splitrecord(l)
            condor[jobid].terminate = ts
        elif l.startswith("016"):
            # post script terminated
            jobid, ts = splitrecord(l)
            condor[jobid].post = ts

def outfiles(dirname):
    for f in os.listdir(dirname):
        path = os.path.join(dirname, f)
        if os.path.isfile(path):
            m = re.match("^(.+)\.out(\.([0-9]+))?$", path)
            if m:
                yield path, os.path.basename(m.group(1)), m.group(3)
        elif os.path.isdir(path):
            for outfile in outfiles(path):
                yield outfile

def parse_invocations(filename):
    invs = []
    f = open(filename, "r")
    cpu = None
    cores = None

    # XXX There is only one slot per file, but that's only true for non-PMC jobs
    # For PMC jobs, when that comes up, we can get it from the cluster-task record
    slot = None

    # Get a generator so that we can pull extra lines off if we need to
    x = f.__iter__()

    for l in x:
        l = l.lstrip()
        if l.startswith("<invocation "):
            l = l[:-2] + "/>"
            e = etree.fromstring(l)
            inv = Invocation()
            inv.duration = e.attrib["duration"]
            inv.xform = e.attrib["transformation"]
            inv.derivation = e.attrib["derivation"]
            inv.site = e.attrib["resource"]
            inv.host = e.attrib["hostaddr"]
            inv.start = utils.epochdate(e.attrib["start"])
            inv.cpu = cpu
            inv.cores = cores
            inv.sent = 0
            inv.received = 0
            inv.procs = []
            inv.files = []
            inv.arguments = []
            inv.connections = []
            inv.slot = slot

            # XXX Skip auxilliary jobs
            if inv.xform.startswith("pegasus"):
                continue
            if inv.xform.endswith("s3cmd"):
                continue
            if inv.xform in ["rm","mkdir"]:
                continue

            invs.append(inv)
        elif l.startswith("<proc ppid"):
            if l.rstrip().endswith("/>"):
                e = etree.fromstring(l)
                inv.procs.append(e)
            else:
                e = etree.fromstring("".join([l, "</proc>"]))
                inv.procs.append(e)

                while True:
                    l = x.next()
                    if l.lstrip().startswith("<file "):
                        e = etree.fromstring(l)
                        use = File()
                        use.name = e.attrib["name"]
                        use.size = int(e.attrib["size"])
                        use.bread = int(e.attrib["bread"])
                        use.bwrite = int(e.attrib["bwrite"])
                        inv.files.append(use)
                    elif l.lstrip().startswith("<socket "):
                        e = etree.fromstring(l)
                        address = e.attrib["address"]
                        port = e.attrib["port"]
                        bsend = e.attrib["bsend"]
                        brecv = e.attrib["brecv"]
                        inv.sent += int(bsend)
                        inv.received += int(brecv)
                    else:
                        break
        elif l.startswith("<status"):
            e = etree.fromstring(l)
            inv.exitcode = e.attrib["raw"]
        elif l.startswith("<cpu"):
            e = etree.fromstring(l)
            cpu = e.text
            cores = e.attrib["count"]
            inv.cpu = cpu
            inv.cores = cores
        elif l.startswith("<arg nr"):
            e = etree.fromstring(l)
            inv.arguments.append(e.text.strip())
        elif l.startswith("<env "):
            if "_CONDOR_SLOT" in l:
                e = etree.fromstring(l)
                inv.slot = e.text
                slot = inv.slot
        elif l.startswith("<usage"):
            # There are two usage records, the first is inside <mainjob>
            # That's the one we want. We skip the other one.
            if hasattr(inv, "utime") or hasattr(inv, "stime"):
                continue

            usage = etree.fromstring(l)
            inv.utime = usage.attrib["utime"]
            inv.stime = usage.attrib["stime"]
        elif l.startswith("<cwd>"):
            e = etree.fromstring(l)
            inv.cwd = e.text


    f.close()

    files = {}
    for inv in invs:
        inv.proc_utime = 0.0
        inv.proc_stime = 0.0
        inv.rsspeak = 0.0
        inv.vmpeak = 0.0
        inv.rbytes = 0
        inv.wbytes = 0

        # XXX This isn't strictly correct, we need to consider overlapping processes for memory
        for p in inv.procs:
            inv.proc_utime += float(p.attrib["utime"])
            inv.proc_stime += float(p.attrib["stime"])
            inv.rsspeak += float(p.attrib["rsspeak"])
            inv.vmpeak += float(p.attrib["vmpeak"])
            inv.rbytes += int(p.attrib["rchar"])
            inv.wbytes += int(p.attrib["wchar"])

        inv.procs = len(inv.procs) > 0

        # Keep track of sizes of files relative to CWD
        newfiles = []
        for f in inv.files:
            if f.name.startswith(inv.cwd):
                f.name = f.name[len(inv.cwd)+1:]
                files[f.name] = f.size
                newfiles.append(f)
        inv.files = newfiles

    return files, invs

def parse_submit_dir(submitdir):
    print "Parsing invocation records..."
    invocations = {}
    files = {}

    for path, jobname, retry in outfiles(submitdir):
        job_files, invs = parse_invocations(path)
        for i in invs:
            i.retry = retry
            i.jobname = jobname
        invocations[jobname] = invs

        files.update(job_files)

    finallist = []
    for jobname in invocations:
        for inv in invocations[jobname]:
            if int(inv.exitcode) != 0:
                print "ERROR: Invocation exitcode != 0 in %s (%s)" % (inv.jobname, inv.exitcode)
                exit(1)
            finallist.append(inv)

    print "%d invocations" % len(finallist)

    return files, finallist

def merge_by_derivation(dax, invocations):
    print "Merging by derivation..."
    derivations = {}
    for i in invocations:
        if i.derivation is None or i.derivation == "":
            print "ERROR: Derivation missing for invocation"
            exit(1)
        derivations[i.derivation] = i
    for j in dax.tasks.values():
        if j.derivation is None or j.derivation == "":
            print "ERROR: Derivation missing for job %s" % j.id
            exit(1)
        if j.derivation in derivations:
            j.invocation = derivations[j.derivation]
        elif j.id in derivations:
            j.invocation = derivations[j.id]
        else:
            print "ERROR: Invocation missing for job %s" % j.id
            exit(1)

def merge_by_transformation(dax, invocations):
    print "Merging by transformation..."
    xforms = {}
    for i in invocations:
        if i.xform not in xforms:
            xforms[i.xform] = []
        xforms[i.xform].append(i)
    for j in dax.tasks.values():
        if j.xform not in xforms:
            print "ERROR: Transformation missing for %s" % j.xform
            exit(1)
        xform = xforms[j.xform]
        if len(xform) == 0:
            print "ERROR: Not enough transformations of type %s" % j.xform
            exit(1)
        j.invocation = xform.pop()

def merge_by_arguments(dax, invocations):
    print "Merging by arguments..."
    args = {}
    for i in invocations:
        args[" ".join(i.arguments)] = i
    for j in dax.tasks.values():
        arguments = " ".join(j.arguments)
        if arguments not in args:
            print "ERROR: No matching arguments for task %s: %s" % (j.id, arguments)
            exit(1)
        j.invocation = args[arguments]

def parse_metadata(metafile):
    metadata = {}
    f = open(metafile, "r")
    for l in f:
        l = l.strip()
        if len(l) == 0:
            continue
        if l[0] == "#":
            continue
        rec = l.split("=")
        key = rec[0].strip()
        value = rec[1].strip()
        metadata[key] = value
    f.close()
    return metadata

def write_output_dax(dax, metadata, files, outfile):
    out = open(outfile, "w")
    out.write('<workflow name="%s" tasks="%d">\n' % (dax.name, len(dax.tasks)))
    if len(metadata) > 0:
        out.write('    <metadata>\n')
        for key, value in metadata.items():
            out.write('        <item key="%s">%s</item>\n' % (key, value))
        out.write('    </metadata>\n')
    for t in dax.tasks.values():
        i = t.invocation
        out.write('    <task id="%s" type="%s" duration="%s" job="%s" site="%s" host="%s" cpu="%s"' \
                % (t.id, t.xform, i.duration, i.jobname, i.site, i.host, i.cpu))
        if i.slot is not None:
            out.write(' slot="%s"' % i.slot)
        if i.procs:
            out.write(' utime="%f" stime="%f"' % (i.proc_utime, i.proc_stime))
        else:
            if hasattr(i, "utime"):
                out.write(' utime="%s"' % i.utime)
            if hasattr(i, "stime"):
                out.write(' stime="%s"' % i.stime)
        out.write('>\n')
        out.write('        <arguments>%s</arguments>\n' % " ".join(t.arguments))
        for f in t.files:
            out.write('        <file name="%s" link="%s"' % (f.name, f.link))
            if f.name in files:
                out.write(' size="%s"' % files[f.name])
            out.write('/>\n')
        out.write('    </task>\n')
    for parent, child in dax.edges:
        out.write('    <edge parent="%s" child="%s"/>\n' % (parent.id, child.id))
    out.write("</workflow>\n")
    out.close()

def write_output_dag(dag, metadata, outfile):
    out = open(outfile, "w")
    out.write('<workflow name="%s" tasks="%d">\n' % (dag.name, len(dag.jobs)))
    if len(metadata) > 0:
        out.write('    <metadata>\n')
        for key, value in metadata.items():
            out.write('        <item key="%s">%s</item>\n' % (key, value))
        out.write('    </metadata>\n')
    for j in dag.jobs.values():
        out.write('    <job name="%s" submit="%s" execute="%s" terminate="%s"'\
                % (j.name, j.submit, j.execute, j.terminate))
        if hasattr(j, "post"):
            out.write(' post="%s"' % j.post)
        if hasattr(j, "priority"):
            out.write(' priority="%s"' % j.priority)
        out.write('/>\n')
    for parent, child in dag.edges:
        out.write('    <edge parent="%s" child="%s"/>\n' % (parent.name, child.name))
    out.write("</workflow>\n")
    out.close()

def write_output_dvdt(daxfile, dax, braindump, outdir, wfname, files):
    dagfile = os.path.join(outdir, "dag")
    workflowfile = os.path.join(outdir, "workflow")

    specification = wfname + ".dax"

    # Generate the workflow file
    wf = open(workflowfile, "w")
    wf.write("name: %s\n" % wfname)
    wf.write("type: pegasus\n")
    wf.write("specification: %s\n" % specification)
    wf.write("dependencies: dag\n")
    # XXX We need to extract the sites from the pegasus command
    #wf.write("batch: %s\n" % braindump["site"])
    wf.write("command: pegasus-plan %s\n" % braindump["planner_arguments"][1:-1].strip())
    wf.write("version: %s\n" % braindump["planner_version"])
    wf.close()

    # Generate the dag file
    df = open(dagfile, "w")
    for parent, child in dax.edges:
        df.write("PARENT %s CHILD %s\n" % (parent.id, child.id))
    wf.close()

    # We group the tasks by job
    jobs = {}
    for tid, t in dax.tasks.items():
        name = t.invocation.jobname
        if name not in jobs:
            jobs[name] = []
        jobs[name].append(t)

    # Then generate a summary file for each job
    for name, j in jobs.items():
        summaryfile = os.path.join(outdir, "%s.summary" % name)
        sf = open(summaryfile, "w")
        for t in j:
            inv = t.invocation
            sf.write("summary:\n")
            sf.write("task_id:                  %s\n" % (t.id))
            sf.write("category:                 %s\n" % (t.xform))
            sf.write("command:                  %s\n" % (t.xform + " " + " ".join(t.arguments)))
            sf.write("start:                    %f s\n" % inv.start)
            sf.write("end:                      %f s\n" % (inv.start + float(inv.duration)))
            sf.write("exit_type:                %s\n" % "normal")
            sf.write("exit_status:              %s\n" % inv.exitcode)
            sf.write("wall_time:                %s s\n" % inv.duration)
            sf.write("max_concurrent_processes: %d procs\n" % 1)
            sf.write("total_processes:          %d procs\n" % 1)
            sf.write("cpu_time:                 %f s\n" % (inv.proc_utime + inv.proc_stime))
            sf.write("virtual_memory:           %f MB\n" % (float(inv.vmpeak)/1024.0))
            sf.write("resident_memory:          %f MB\n" % (float(inv.rsspeak)/1024.0))
            sf.write("swap_memory:              %d MB\n" % 0)
            sf.write("bytes_read:               %d B\n" % inv.rbytes)
            sf.write("bytes_written:            %d B\n" % inv.wbytes)
            sf.write("sent:                     %d B\n" % inv.sent)
            sf.write("received:                 %d B\n" % inv.received)

            for f in inv.files:
                if f.bread > 0:
                    sf.write("input:                    %s\n" % f.name)
                if f.bwrite > 0:
                    sf.write("output:                   %s\n" % f.name)
                if f.bread == 0 and f.bwrite == 0:
                    # Try to find it in the dax files
                    for g in t.files:
                        if f.name == g.name:
                            if g.link == "output":
                                sf.write("output:                   %s\n" % f.name)
                            elif g.link == "input":
                                sf.write("input:                    %s\n" % f.name)
                            elif g.link in ["inout", "checkpoint"]:
                                sf.write("input:                    %s\n" % f.name)
                                sf.write("output:                   %s\n" % f.name)
                            else:
                                raise Exception("Unexpected link value for %s: %s" % (f.name, f.link))
                            break

            sf.write("\n")
        sf.close()

    # Generate a 'files' file
    ff = open(os.path.join(outdir, "files"), "w")
    for f in files:
        ff.write("name: %s\n" % f)
        ff.write("size: %d B\n\n" % files[f])
    ff.close()

    # Copy the dax file
    shutil.copy(daxfile, os.path.join(outdir, specification))

def parse_condor_timestamp(ts):
    dt = datetime.datetime.strptime(ts, "%m/%d/%Y %H:%M:%S")
    return int(dt.strftime("%s"))

def get_makespan(dagman_log):
    f = open(dagman_log, "r")

    makespan = 0
    for l in f:
        if l.startswith("001"):
            ts = splitrecord(l)[1]
            start = parse_condor_timestamp(ts)
        elif l.startswith("005"):
            ts = splitrecord(l)[1]
            end = parse_condor_timestamp(ts)
            makespan += end - start

    f.close()

    return makespan

def parse_file_sizes(file_sizes):
    files = {}
    f = open(file_sizes, "r")
    try:
        for l in f:
            filename, size = l.split()
            files[filename] = size
    finally:
        f.close()

    return files

def main():
    parser = OptionParser(usage="Usage: %prog [options] SUBMIT_DIR")
    parser.add_option("-d", "--dax", dest="dax", action="store", default=None,
                      help="DAX file. Default: find in braindump.txt")
    parser.add_option("-D", "--dag", dest="dag", action="store", default=None,
                      help="DAG file. Default: find in braindump.txt")
    parser.add_option("-l", "--log", dest="log", action="store", default=None,
                      help="Condor job log. Default: find in braindump.txt")
    parser.add_option("-t", dest="merge_by_transformation", action="store_true", default=False,
                      help="Merge the DAX and invocation records by transformation. "
                      "This method is inaccurate because there is no guarantee that "
                      "an invocation will be matched to the corresponding job from "
                      "the DAX exactly. The default is to use derivations, which "
                      "is accurate, but only works for workflows that used dv-name "
                      "or workflows run using Pegasus 4 or later.")
    parser.add_option("-a", dest="merge_by_arguments", action="store_true", default=False,
                      help="Merge the DAX and invocation records using their arguments. "
                      "This method is better than merging by transformation, but not as "
                      "good as merging by dervation, which is the default, but doesn't work "
                      "for all workflows.")
    parser.add_option("-m", dest="metadata", action="append",
                      help="Add a metadata file with key=value pairs for this workflow")
    parser.add_option("-f", "--format", dest="format", action="store", default="XML",
                      help="Output format ('XML' or 'dvdt'). Default: XML")
    parser.add_option("-o", "--output", dest="output", action="store", default=None,
                      help="Output location. Default: SUBMIT_DIR/package[/WFNAME.xml]")
    parser.add_option("-n", "--name", dest="wfname", action="store", default=None,
                      help="Workflow name. Default: extract from braindump")
    parser.add_option("-s", "--sizes", dest="file_sizes", action="store", default=None,
                      help="File sizes")

    options, args = parser.parse_args()

    if len(args) > 1:
        parser.error("Invalid argument")
    if len(args) < 1:
        parser.error("Specify SUBMIT_DIR")

    submitdir = args[0]

    if not os.path.isdir(submitdir):
        parser.error("Invalid Submit directory: %s" % submitdir)

    bdfile = os.path.join(submitdir, "braindump.txt")
    braindump = Braindump(bdfile)

    if options.dax is not None:
        daxfile = options.dax
    else:
        daxfile = braindump.get_path("dax")

    if not os.path.isfile(daxfile):
        parser.error("DAX file not found: %s" % daxfile)

    if options.dag is not None:
        dagfile = options.dag
    else:
        dagfile = braindump.get_path("dag")

    if not os.path.isfile(dagfile):
        parser.error("DAG file not found: %s" % dagfile)

    if options.log is not None:
        logfile = options.log
    else:
        logfile = braindump.get_path("condor_log")
        if logfile is None:
            logfile = dagfile.replace(".dag", ".log")

    if not os.path.isfile(logfile):
        parser.error("Condor log file not found: %s" % logfile)

    dagman_log = dagfile + ".dagman.log"
    if not os.path.isfile(dagman_log):
        parser.error("DAGMan log file not found: %s" % dagman_log)

    if options.format not in ["XML","dvdt"]:
        parser.error("Invalid format: %s" % options.format)

    if options.merge_by_transformation and options.merge_by_arguments:
        parser.error("Specify -t or -a, but not both. -a is better.")

    if options.wfname is None:
        wfname = braindump["dax_label"]
        if wfname is None:
            wfname = braindump["pegasus_wf_name"]
    else:
        wfname = options.wfname

    outdir = options.output

    if outdir is None:
        outdir = os.path.join(submitdir, "package")

    if os.path.isfile(outdir):
        parser.error("Output directory is a file: %s" % outdir)

    if not os.path.isdir(outdir):
        os.makedirs(outdir)

    makespan = get_makespan(dagman_log)
    dag = parse_dag(dagfile)
    parse_job_log(logfile, dag)
    files, invocations = parse_submit_dir(submitdir)
    dax = parse_dax(daxfile)

    if len(dax.tasks) != len(invocations):
        print "ERROR: # DAX tasks != # invocations"
        exit(1)

    if options.merge_by_arguments:
        merge_by_arguments(dax, invocations)
    elif options.merge_by_transformation:
        merge_by_transformation(dax, invocations)
    else:
        merge_by_derivation(dax, invocations)

    metadata = {}
    if options.metadata:
        for f in options.metadata:
            metadata.update(parse_metadata(f))

    metadata["makespan"] = makespan

    if options.file_sizes:
        files = parse_file_sizes(options.file_sizes)

    if options.format == "XML":
        basename = os.path.join(outdir, braindump["pegasus_wf_name"])
        write_output_dax(dax, metadata, files, basename + ".wf")
        write_output_dag(dag, metadata, basename + "_dag.wf")
    elif options.format == "dvdt":
        write_output_dvdt(daxfile, dax, braindump, outdir, wfname, files)

if __name__ == '__main__':
    main()

