#! /usr/bin/env python # encoding: utf-8 import sys if sys.hexversion < 0x020400f0: from sets import Set as set import os,shutil,sys,re,random,datetime,tempfile,shlex from Utils import md5 import Build,Runner,Utils,Node,Logs,Options from Logs import debug,warn,error from Constants import* algotype=NORMAL COMPILE_TEMPLATE_SHELL=''' def f(task): env = task.env wd = getattr(task, 'cwd', None) p = env.get_flat cmd = \'\'\' %s \'\'\' % s return task.exec_command(cmd, cwd=wd) ''' COMPILE_TEMPLATE_NOSHELL=''' def f(task): env = task.env wd = getattr(task, 'cwd', None) def to_list(xx): if isinstance(xx, str): return [xx] return xx lst = [] %s lst = [x for x in lst if x] return task.exec_command(lst, cwd=wd) ''' file_deps=Utils.nada class TaskManager(object): def __init__(self): self.groups=[] self.tasks_done=[] self.current_group=0 self.groups_names={} def group_name(self,g): if not isinstance(g,TaskGroup): g=self.groups[g] for x in self.groups_names: if id(self.groups_names[x])==id(g): return x return'' def group_idx(self,tg): se=id(tg) for i in range(len(self.groups)): g=self.groups[i] for t in g.tasks_gen: if id(t)==se: return i return None def get_next_set(self): ret=None while not ret and self.current_group0: self.set_order(keys[i],keys[j]) elif val<0: self.set_order(keys[j],keys[i]) def tasks_in_parallel(self): if not self.ready:self.prepare() keys=self.cstr_groups.keys() unconnected=[] remainder=[] for u in keys: for k in self.cstr_order.values(): if u in k: remainder.append(u) break else: unconnected.append(u) toreturn=[] for y in unconnected: toreturn.extend(self.cstr_groups[y]) for y in unconnected: try:self.cstr_order.__delitem__(y) except KeyError:pass self.cstr_groups.__delitem__(y) if not toreturn and remainder: raise Utils.WafError("circular order constraint detected %r"%remainder) return toreturn def tasks_by_max_jobs(self): if not self.ready:self.prepare() if not self.temp_tasks:self.temp_tasks=self.tasks_in_parallel() if not self.temp_tasks:return(None,None) maxjobs=MAXJOBS ret=[] remaining=[] for t in self.temp_tasks: m=getattr(t,"maxjobs",getattr(self.__class__,"maxjobs",MAXJOBS)) if m>maxjobs: remaining.append(t) elif m task failed (err #%d): %r"%(self.err_code,self) except AttributeError: return" -> task failed: %r"%self elif self.hasrun==MISSING: return" -> missing files: %r"%self else: return'' def install(self): bld=self.generator.bld d=self.attr('install') if self.attr('install_path'): lst=[a.relpath_gen(bld.srcnode)for a in self.outputs] perm=self.attr('chmod',O644) if self.attr('src'): lst+=[a.relpath_gen(bld.srcnode)for a in self.inputs] if self.attr('filename'): dir=self.install_path.rstrip(os.sep)+os.sep+self.attr('filename') bld.install_as(dir,lst[0],self.env,perm) else: bld.install_files(self.install_path,lst,self.env,perm) class Task(TaskBase): vars=[] def __init__(self,env,**kw): TaskBase.__init__(self,**kw) self.env=env self.inputs=[] self.outputs=[] self.dep_nodes=[] self.run_after=[] def __str__(self): env=self.env src_str=' '.join([a.nice_path(env)for a in self.inputs]) tgt_str=' '.join([a.nice_path(env)for a in self.outputs]) if self.outputs:sep=' -> ' else:sep='' return'%s: %s%s%s\n'%(self.__class__.__name__.replace('_task',''),src_str,sep,tgt_str) def __repr__(self): return"".join(['\n\t{task: ',self.__class__.__name__," ",",".join([x.name for x in self.inputs])," -> ",",".join([x.name for x in self.outputs]),'}']) def unique_id(self): try: return self.uid except AttributeError: m=md5() up=m.update up(self.__class__.__name__) up(self.env.variant()) p=None for x in self.inputs+self.outputs: if p!=x.parent.id: p=x.parent.id up(x.parent.abspath()) up(x.name) self.uid=m.digest() return self.uid def set_inputs(self,inp): if isinstance(inp,list):self.inputs+=inp else:self.inputs.append(inp) def set_outputs(self,out): if isinstance(out,list):self.outputs+=out else:self.outputs.append(out) def set_run_after(self,task): assert isinstance(task,TaskBase) self.run_after.append(task) def add_file_dependency(self,filename): node=self.generator.bld.path.find_resource(filename) self.dep_nodes.append(node) def signature(self): try:return self.cache_sig[0] except AttributeError:pass self.m=md5() exp_sig=self.sig_explicit_deps() var_sig=self.sig_vars() imp_sig=SIG_NIL if self.scan: try: imp_sig=self.sig_implicit_deps() except ValueError: return self.signature() ret=self.m.digest() self.cache_sig=(ret,exp_sig,imp_sig,var_sig) return ret def runnable_status(self): if self.inputs and(not self.outputs): if not getattr(self.__class__,'quiet',None): warn("invalid task (no inputs OR outputs): override in a Task subclass or set the attribute 'quiet' %r"%self) for t in self.run_after: if not t.hasrun: return ASK_LATER env=self.env bld=self.generator.bld new_sig=self.signature() key=self.unique_id() try: prev_sig=bld.task_sigs[key][0] except KeyError: debug("task: task %r must run as it was never run before or the task code changed",self) return RUN_ME for node in self.outputs: variant=node.variant(env) try: if bld.node_sigs[variant][node.id]!=new_sig: return RUN_ME except KeyError: debug("task: task %r must run as the output nodes do not exist",self) return RUN_ME if Logs.verbose:self.debug_why(bld.task_sigs[key]) if new_sig!=prev_sig: return RUN_ME return SKIP_ME def post_run(self): bld=self.generator.bld env=self.env sig=self.signature() ssig=sig.encode('hex') variant=env.variant() for node in self.outputs: try: os.stat(node.abspath(env)) except OSError: self.hasrun=MISSING self.err_msg='-> missing file: %r'%node.abspath(env) raise Utils.WafError bld.node_sigs[variant][node.id]=sig bld.task_sigs[self.unique_id()]=self.cache_sig if not Options.cache_global or Options.options.nocache or not self.outputs: return None if getattr(self,'cached',None): return None dname=os.path.join(Options.cache_global,ssig) tmpdir=tempfile.mkdtemp(prefix=Options.cache_global+os.sep+'waf') try: shutil.rmtree(dname) except: pass try: i=0 for node in self.outputs: variant=node.variant(env) dest=os.path.join(tmpdir,str(i)+node.name) shutil.copy2(node.abspath(env),dest) i+=1 except(OSError,IOError): try: shutil.rmtree(tmpdir) except: pass else: try: os.rename(tmpdir,dname) except OSError: try: shutil.rmtree(tmpdir) except: pass else: try: os.chmod(dname,O755) except: pass def can_retrieve_cache(self): if not Options.cache_global or Options.options.nocache or not self.outputs: return None env=self.env sig=self.signature() ssig=sig.encode('hex') dname=os.path.join(Options.cache_global,ssig) try: t1=os.stat(dname).st_mtime except OSError: return None i=0 for node in self.outputs: variant=node.variant(env) orig=os.path.join(dname,str(i)+node.name) try: shutil.copy2(orig,node.abspath(env)) os.utime(orig,None) except(OSError,IOError): debug('task: failed retrieving file') return None i+=1 try: t2=os.stat(dname).st_mtime except OSError: return None if t1!=t2: return None for node in self.outputs: self.generator.bld.node_sigs[variant][node.id]=sig if Options.options.progress_bar<1: self.generator.bld.printout('restoring from cache %r\n'%node.bldpath(env)) self.cached=True return 1 def debug_why(self,old_sigs): new_sigs=self.cache_sig def v(x): return x.encode('hex') debug("Task %r",self) msgs=['Task must run','* Source file or manual dependency','* Implicit dependency','* Environment variable'] tmp='task: -> %s: %s %s' for x in xrange(len(msgs)): if(new_sigs[x]!=old_sigs[x]): debug(tmp,msgs[x],v(old_sigs[x]),v(new_sigs[x])) def sig_explicit_deps(self): bld=self.generator.bld up=self.m.update for x in self.inputs+getattr(self,'dep_nodes',[]): if not x.parent.id in bld.cache_scanned_folders: bld.rescan(x.parent) variant=x.variant(self.env) try: up(bld.node_sigs[variant][x.id]) except KeyError: raise Utils.WafError('Missing node signature for %r (required by %r)'%(x,self)) if bld.deps_man: additional_deps=bld.deps_man for x in self.inputs+self.outputs: try: d=additional_deps[x.id] except KeyError: continue for v in d: if isinstance(v,Node.Node): bld.rescan(v.parent) variant=v.variant(self.env) try: v=bld.node_sigs[variant][v.id] except KeyError: raise Utils.WafError('Missing node signature for %r (required by %r)'%(v,self)) elif hasattr(v,'__call__'): v=v() up(v) for x in self.dep_nodes: v=bld.node_sigs[x.variant(self.env)][x.id] up(v) return self.m.digest() def sig_vars(self): bld=self.generator.bld env=self.env act_sig=bld.hash_env_vars(env,self.__class__.vars) self.m.update(act_sig) dep_vars=getattr(self,'dep_vars',None) if dep_vars: self.m.update(bld.hash_env_vars(env,dep_vars)) return self.m.digest() scan=None def sig_implicit_deps(self): bld=self.generator.bld key=self.unique_id() prev_sigs=bld.task_sigs.get(key,()) if prev_sigs: try: if prev_sigs[2]==self.compute_sig_implicit_deps(): return prev_sigs[2] except(KeyError,OSError): pass del bld.task_sigs[key] raise ValueError('rescan') (nodes,names)=self.scan() if Logs.verbose: debug('deps: scanner for %s returned %s %s',str(self),str(nodes),str(names)) bld.node_deps[key]=nodes bld.raw_deps[key]=names try: sig=self.compute_sig_implicit_deps() except KeyError: try: nodes=[] for k in bld.node_deps.get(self.unique_id(),[]): if k.id&3==2: if not k.id in bld.node_sigs[0]: nodes.append(k) else: if not k.id in bld.node_sigs[self.env.variant()]: nodes.append(k) except: nodes='?' raise Utils.WafError('Missing node signature for %r (for implicit dependencies %r)'%(nodes,self)) return sig def compute_sig_implicit_deps(self): upd=self.m.update bld=self.generator.bld tstamp=bld.node_sigs env=self.env for k in bld.node_deps.get(self.unique_id(),[]): if not k.parent.id in bld.cache_scanned_folders: bld.rescan(k.parent) if k.id&3==2: upd(tstamp[0][k.id]) else: upd(tstamp[env.variant()][k.id]) return self.m.digest() def funex(c): dc={} exec(c,dc) return dc['f'] reg_act=re.compile(r"(?P\\)|(?P\$\$)|(?P\$\{(?P\w+)(?P.*?)\})",re.M) def compile_fun_shell(name,line): extr=[] def repl(match): g=match.group if g('dollar'):return"$" elif g('backslash'):return'\\\\' elif g('subst'):extr.append((g('var'),g('code')));return"%s" return None line=reg_act.sub(repl,line)or line parm=[] dvars=[] app=parm.append for(var,meth)in extr: if var=='SRC': if meth:app('task.inputs%s'%meth) else:app('" ".join([a.srcpath(env) for a in task.inputs])') elif var=='TGT': if meth:app('task.outputs%s'%meth) else:app('" ".join([a.bldpath(env) for a in task.outputs])') else: if not var in dvars:dvars.append(var) app("p('%s')"%var) if parm:parm="%% (%s) "%(',\n\t\t'.join(parm)) else:parm='' c=COMPILE_TEMPLATE_SHELL%(line,parm) debug('action: %s',c) return(funex(c),dvars) def compile_fun_noshell(name,line): extr=[] def repl(match): g=match.group if g('dollar'):return"$" elif g('subst'):extr.append((g('var'),g('code')));return"<<|@|>>" return None line2=reg_act.sub(repl,line) params=line2.split('<<|@|>>') buf=[] dvars=[] app=buf.append for x in xrange(len(extr)): params[x]=params[x].strip() if params[x]: app("lst.extend(%r)"%params[x].split()) (var,meth)=extr[x] if var=='SRC': if meth:app('lst.append(task.inputs%s)'%meth) else:app("lst.extend([a.srcpath(env) for a in task.inputs])") elif var=='TGT': if meth:app('lst.append(task.outputs%s)'%meth) else:app("lst.extend([a.bldpath(env) for a in task.outputs])") else: app('lst.extend(to_list(env[%r]))'%var) if not var in dvars:dvars.append(var) if params[-1]: app("lst.extend(%r)"%shlex.split(params[-1])) fun=COMPILE_TEMPLATE_NOSHELL%"\n\t".join(buf) debug('action: %s',fun) return(funex(fun),dvars) def compile_fun(name,line,shell=None): if line.find('<')>0 or line.find('>')>0 or line.find('&&')>0: shell=True if shell is None: if sys.platform=='win32': shell=False else: shell=True if shell: return compile_fun_shell(name,line) else: return compile_fun_noshell(name,line) def simple_task_type(name,line,color='GREEN',vars=[],ext_in=[],ext_out=[],before=[],after=[],shell=None): (fun,dvars)=compile_fun(name,line,shell) fun.code=line return task_type_from_func(name,fun,vars or dvars,color,ext_in,ext_out,before,after) def task_type_from_func(name,func,vars=[],color='GREEN',ext_in=[],ext_out=[],before=[],after=[]): params={'run':func,'vars':vars,'color':color,'name':name,'ext_in':Utils.to_list(ext_in),'ext_out':Utils.to_list(ext_out),'before':Utils.to_list(before),'after':Utils.to_list(after),} cls=type(Task)(name,(Task,),params) TaskBase.classes[name]=cls return cls def always_run(cls): old=cls.runnable_status def always(self): ret=old(self) if ret==SKIP_ME: return RUN_ME return ret cls.runnable_status=always def update_outputs(cls): old_post_run=cls.post_run def post_run(self): old_post_run(self) bld=self.generator.bld for output in self.outputs: bld.node_sigs[self.env.variant()][output.id]=Utils.h_file(output.abspath(self.env)) cls.post_run=post_run old_runnable_status=cls.runnable_status def runnable_status(self): status=old_runnable_status(self) if status!=RUN_ME: return status try: bld=self.outputs[0].__class__.bld new_sig=self.signature() prev_sig=bld.task_sigs[self.unique_id()][0] if prev_sig==new_sig: for x in self.outputs: if not x.id in bld.node_sigs[self.env.variant()]: return RUN_ME return SKIP_ME except KeyError: pass except IndexError: pass return RUN_ME cls.runnable_status=runnable_status def extract_outputs(tasks): v={} for x in tasks: try: (ins,outs)=v[x.env.variant()] except KeyError: ins={} outs={} v[x.env.variant()]=(ins,outs) for a in getattr(x,'inputs',[]): try:ins[a.id].append(x) except KeyError:ins[a.id]=[x] for a in getattr(x,'outputs',[]): try:outs[a.id].append(x) except KeyError:outs[a.id]=[x] for(ins,outs)in v.values(): links=set(ins.iterkeys()).intersection(outs.iterkeys()) for k in links: for a in ins[k]: for b in outs[k]: a.set_run_after(b) def extract_deps(tasks): extract_outputs(tasks) out_to_task={} for x in tasks: v=x.env.variant() try: lst=x.outputs except AttributeError: pass else: for node in lst: out_to_task[(v,node.id)]=x dep_to_task={} for x in tasks: try: x.signature() except: pass v=x.env.variant() key=x.unique_id() for k in x.generator.bld.node_deps.get(x.unique_id(),[]): try:dep_to_task[(v,k.id)].append(x) except KeyError:dep_to_task[(v,k.id)]=[x] deps=set(dep_to_task.keys()).intersection(set(out_to_task.keys())) for idx in deps: for k in dep_to_task[idx]: k.set_run_after(out_to_task[idx]) for x in tasks: try: delattr(x,'cache_sig') except AttributeError: pass