midori/waf-modules/wafadmin/Build.py
2013-10-23 20:26:27 -07:00

676 lines
20 KiB
Python

#! /usr/bin/env python
# encoding: utf-8
import sys
if sys.hexversion < 0x020400f0: from sets import Set as set
import os,sys,errno,re,glob,gc,datetime,shutil
try:import cPickle
except:import pickle as cPickle
import Runner,TaskGen,Node,Scripting,Utils,Environment,Task,Logs,Options
from Logs import debug,error,info
from Constants import*
SAVED_ATTRS='root srcnode bldnode node_sigs node_deps raw_deps task_sigs id_nodes'.split()
bld=None
class BuildError(Utils.WafError):
def __init__(self,b=None,t=[]):
self.bld=b
self.tasks=t
self.ret=1
Utils.WafError.__init__(self,self.format_error())
def format_error(self):
lst=['Build failed:']
for tsk in self.tasks:
txt=tsk.format_error()
if txt:lst.append(txt)
sep=' '
if len(lst)>2:
sep='\n'
return sep.join(lst)
def group_method(fun):
def f(*k,**kw):
if not k[0].is_install:
return False
postpone=True
if'postpone'in kw:
postpone=kw['postpone']
del kw['postpone']
if postpone:
m=k[0].task_manager
if not m.groups:m.add_group()
m.groups[m.current_group].post_funs.append((fun,k,kw))
if not'cwd'in kw:
kw['cwd']=k[0].path
else:
fun(*k,**kw)
return f
class BuildContext(Utils.Context):
def __init__(self):
global bld
bld=self
self.task_manager=Task.TaskManager()
self.id_nodes=0
self.idx={}
self.all_envs={}
self.bdir=''
self.path=None
self.deps_man=Utils.DefaultDict(list)
self.cache_node_abspath={}
self.cache_scanned_folders={}
self.uninstall=[]
for v in'cache_node_abspath task_sigs node_deps raw_deps node_sigs'.split():
var={}
setattr(self,v,var)
self.cache_dir_contents={}
self.all_task_gen=[]
self.task_gen_cache_names={}
self.cache_sig_vars={}
self.log=None
self.root=None
self.srcnode=None
self.bldnode=None
class node_class(Node.Node):
pass
self.node_class=node_class
self.node_class.__module__="Node"
self.node_class.__name__="Nodu"
self.node_class.bld=self
self.is_install=None
def __copy__(self):
raise Utils.WafError('build contexts are not supposed to be cloned')
def load(self):
try:
env=Environment.Environment(os.path.join(self.cachedir,'build.config.py'))
except(IOError,OSError):
pass
else:
if env['version']<HEXVERSION:
raise Utils.WafError('Version mismatch! reconfigure the project')
for t in env['tools']:
self.setup(**t)
try:
gc.disable()
f=data=None
Node.Nodu=self.node_class
try:
f=open(os.path.join(self.bdir,DBFILE),'rb')
except(IOError,EOFError):
pass
try:
if f:data=cPickle.load(f)
except AttributeError:
if Logs.verbose>1:raise
if data:
for x in SAVED_ATTRS:setattr(self,x,data[x])
else:
debug('build: Build cache loading failed')
finally:
if f:f.close()
gc.enable()
def save(self):
gc.disable()
self.root.__class__.bld=None
Node.Nodu=self.node_class
db=os.path.join(self.bdir,DBFILE)
file=open(db+'.tmp','wb')
data={}
for x in SAVED_ATTRS:data[x]=getattr(self,x)
cPickle.dump(data,file,-1)
file.close()
try:os.unlink(db)
except OSError:pass
os.rename(db+'.tmp',db)
self.root.__class__.bld=self
gc.enable()
def clean(self):
debug('build: clean called')
precious=set([])
for env in self.all_envs.values():
for x in env[CFG_FILES]:
node=self.srcnode.find_resource(x)
if node:
precious.add(node.id)
def clean_rec(node):
for x in list(node.childs.keys()):
nd=node.childs[x]
tp=nd.id&3
if tp==Node.DIR:
clean_rec(nd)
elif tp==Node.BUILD:
if nd.id in precious:continue
for env in self.all_envs.values():
try:os.remove(nd.abspath(env))
except OSError:pass
node.childs.__delitem__(x)
clean_rec(self.srcnode)
for v in'node_sigs node_deps task_sigs raw_deps cache_node_abspath'.split():
setattr(self,v,{})
def compile(self):
debug('build: compile called')
self.flush()
self.generator=Runner.Parallel(self,Options.options.jobs)
def dw(on=True):
if Options.options.progress_bar:
if on:sys.stderr.write(Logs.colors.cursor_on)
else:sys.stderr.write(Logs.colors.cursor_off)
debug('build: executor starting')
back=os.getcwd()
os.chdir(self.bldnode.abspath())
try:
try:
dw(on=False)
self.generator.start()
except KeyboardInterrupt:
dw()
self.save()
raise
except Exception:
dw()
raise
else:
dw()
self.save()
if self.generator.error:
raise BuildError(self,self.task_manager.tasks_done)
finally:
os.chdir(back)
def install(self):
debug('build: install called')
self.flush()
if self.is_install<0:
lst=[]
for x in self.uninstall:
dir=os.path.dirname(x)
if not dir in lst:lst.append(dir)
lst.sort()
lst.reverse()
nlst=[]
for y in lst:
x=y
while len(x)>4:
if not x in nlst:nlst.append(x)
x=os.path.dirname(x)
nlst.sort()
nlst.reverse()
for x in nlst:
try:os.rmdir(x)
except OSError:pass
def new_task_gen(self,*k,**kw):
if self.task_gen_cache_names:
self.task_gen_cache_names={}
kw['bld']=self
if len(k)==0:
ret=TaskGen.task_gen(*k,**kw)
else:
cls_name=k[0]
try:cls=TaskGen.task_gen.classes[cls_name]
except KeyError:raise Utils.WscriptError('%s is not a valid task generator -> %s'%(cls_name,[x for x in TaskGen.task_gen.classes]))
ret=cls(*k,**kw)
return ret
def __call__(self,*k,**kw):
if self.task_gen_cache_names:
self.task_gen_cache_names={}
kw['bld']=self
return TaskGen.task_gen(*k,**kw)
def load_envs(self):
try:
lst=Utils.listdir(self.cachedir)
except OSError,e:
if e.errno==errno.ENOENT:
raise Utils.WafError('The project was not configured: run "waf configure" first!')
else:
raise
if not lst:
raise Utils.WafError('The cache directory is empty: reconfigure the project')
for file in lst:
if file.endswith(CACHE_SUFFIX):
env=Environment.Environment(os.path.join(self.cachedir,file))
name=file[:-len(CACHE_SUFFIX)]
self.all_envs[name]=env
self.init_variants()
for env in self.all_envs.values():
for f in env[CFG_FILES]:
newnode=self.path.find_or_declare(f)
try:
hash=Utils.h_file(newnode.abspath(env))
except(IOError,AttributeError):
error("cannot find "+f)
hash=SIG_NIL
self.node_sigs[env.variant()][newnode.id]=hash
self.bldnode=self.root.find_dir(self.bldnode.abspath())
self.path=self.srcnode=self.root.find_dir(self.srcnode.abspath())
self.cwd=self.bldnode.abspath()
def setup(self,tool,tooldir=None,funs=None):
if isinstance(tool,list):
for i in tool:self.setup(i,tooldir)
return
if not tooldir:tooldir=Options.tooldir
module=Utils.load_tool(tool,tooldir)
if hasattr(module,"setup"):module.setup(self)
def init_variants(self):
debug('build: init variants')
lstvariants=[]
for env in self.all_envs.values():
if not env.variant()in lstvariants:
lstvariants.append(env.variant())
self.lst_variants=lstvariants
debug('build: list of variants is %r',lstvariants)
for name in lstvariants+[0]:
for v in'node_sigs cache_node_abspath'.split():
var=getattr(self,v)
if not name in var:
var[name]={}
def load_dirs(self,srcdir,blddir,load_cache=1):
assert(os.path.isabs(srcdir))
assert(os.path.isabs(blddir))
self.cachedir=os.path.join(blddir,CACHE_DIR)
if srcdir==blddir:
raise Utils.WafError("build dir must be different from srcdir: %s <-> %s "%(srcdir,blddir))
self.bdir=blddir
self.load()
if not self.root:
Node.Nodu=self.node_class
self.root=Node.Nodu('',None,Node.DIR)
if not self.srcnode:
self.srcnode=self.root.ensure_dir_node_from_path(srcdir)
debug('build: srcnode is %s and srcdir %s',self.srcnode.name,srcdir)
self.path=self.srcnode
try:os.makedirs(blddir)
except OSError:pass
if not self.bldnode:
self.bldnode=self.root.ensure_dir_node_from_path(blddir)
self.init_variants()
def rescan(self,src_dir_node):
if self.cache_scanned_folders.get(src_dir_node.id,None):return
self.cache_scanned_folders[src_dir_node.id]=True
if hasattr(self,'repository'):self.repository(src_dir_node)
if not src_dir_node.name and sys.platform=='win32':
return
parent_path=src_dir_node.abspath()
try:
lst=set(Utils.listdir(parent_path))
except OSError:
lst=set([])
self.cache_dir_contents[src_dir_node.id]=lst
cache=self.node_sigs[0]
for x in src_dir_node.childs.values():
if x.id&3!=Node.FILE:continue
if x.name in lst:
try:
cache[x.id]=Utils.h_file(x.abspath())
except IOError:
raise Utils.WafError('The file %s is not readable or has become a dir'%x.abspath())
else:
try:del cache[x.id]
except KeyError:pass
del src_dir_node.childs[x.name]
h1=self.srcnode.height()
h2=src_dir_node.height()
lst=[]
child=src_dir_node
while h2>h1:
lst.append(child.name)
child=child.parent
h2-=1
lst.reverse()
try:
for variant in self.lst_variants:
sub_path=os.path.join(self.bldnode.abspath(),variant,*lst)
self.listdir_bld(src_dir_node,sub_path,variant)
except OSError:
for node in src_dir_node.childs.values():
if node.id&3!=Node.BUILD:
continue
for dct in self.node_sigs.values():
if node.id in dct:
dct.__delitem__(node.id)
src_dir_node.childs.__delitem__(node.name)
for variant in self.lst_variants:
sub_path=os.path.join(self.bldnode.abspath(),variant,*lst)
try:
os.makedirs(sub_path)
except OSError:
pass
def listdir_src(self,parent_node):
pass
def remove_node(self,node):
pass
def listdir_bld(self,parent_node,path,variant):
i_existing_nodes=[x for x in parent_node.childs.values()if x.id&3==Node.BUILD]
lst=set(Utils.listdir(path))
node_names=set([x.name for x in i_existing_nodes])
remove_names=node_names-lst
ids_to_remove=[x.id for x in i_existing_nodes if x.name in remove_names]
cache=self.node_sigs[variant]
for nid in ids_to_remove:
if nid in cache:
cache.__delitem__(nid)
def get_env(self):
return self.env_of_name('default')
def set_env(self,name,val):
self.all_envs[name]=val
env=property(get_env,set_env)
def add_manual_dependency(self,path,value):
if isinstance(path,Node.Node):
node=path
elif os.path.isabs(path):
node=self.root.find_resource(path)
else:
node=self.path.find_resource(path)
self.deps_man[node.id].append(value)
def launch_node(self):
try:
return self.p_ln
except AttributeError:
self.p_ln=self.root.find_dir(Options.launch_dir)
return self.p_ln
def glob(self,pattern,relative=True):
path=self.path.abspath()
files=[self.root.find_resource(x)for x in glob.glob(path+os.sep+pattern)]
if relative:
files=[x.path_to_parent(self.path)for x in files if x]
else:
files=[x.abspath()for x in files if x]
return files
def add_group(self,*k):
self.task_manager.add_group(*k)
def set_group(self,*k,**kw):
self.task_manager.set_group(*k,**kw)
def hash_env_vars(self,env,vars_lst):
idx=str(id(env))+str(vars_lst)
try:return self.cache_sig_vars[idx]
except KeyError:pass
lst=[str(env[a])for a in vars_lst]
ret=Utils.h_list(lst)
debug('envhash: %r %r',ret,lst)
self.cache_sig_vars[idx]=ret
return ret
def name_to_obj(self,name,env):
cache=self.task_gen_cache_names
if not cache:
for x in self.all_task_gen:
vt=x.env.variant()+'_'
if x.name:
cache[vt+x.name]=x
else:
if isinstance(x.target,str):
target=x.target
else:
target=' '.join(x.target)
v=vt+target
if not cache.get(v,None):
cache[v]=x
return cache.get(env.variant()+'_'+name,None)
def flush(self,all=1):
self.ini=datetime.datetime.now()
self.task_gen_cache_names={}
self.name_to_obj('',self.env)
debug('build: delayed operation TaskGen.flush() called')
if Options.options.compile_targets:
debug('task_gen: posting objects %r listed in compile_targets',Options.options.compile_targets)
mana=self.task_manager
to_post=[]
min_grp=0
target_objects=Utils.DefaultDict(list)
for target_name in Options.options.compile_targets.split(','):
target_name=target_name.strip()
for env in self.all_envs.values():
tg=self.name_to_obj(target_name,env)
if tg:
target_objects[target_name].append(tg)
m=mana.group_idx(tg)
if m>min_grp:
min_grp=m
to_post=[tg]
elif m==min_grp:
to_post.append(tg)
if not target_name in target_objects and all:
raise Utils.WafError("target '%s' does not exist"%target_name)
debug('group: Forcing up to group %s for target %s',mana.group_name(min_grp),Options.options.compile_targets)
for i in xrange(len(mana.groups)):
mana.current_group=i
if i==min_grp:
break
g=mana.groups[i]
debug('group: Forcing group %s',mana.group_name(g))
for t in g.tasks_gen:
debug('group: Posting %s',t.name or t.target)
t.post()
for t in to_post:
t.post()
else:
debug('task_gen: posting objects (normal)')
ln=self.launch_node()
if ln.is_child_of(self.bldnode)or not ln.is_child_of(self.srcnode):
ln=self.srcnode
proj_node=self.root.find_dir(os.path.split(Utils.g_module.root_path)[0])
if proj_node.id!=self.srcnode.id:
ln=self.srcnode
for i in xrange(len(self.task_manager.groups)):
g=self.task_manager.groups[i]
self.task_manager.current_group=i
if Logs.verbose:
groups=[x for x in self.task_manager.groups_names if id(self.task_manager.groups_names[x])==id(g)]
name=groups and groups[0]or'unnamed'
Logs.debug('group: group',name)
for tg in g.tasks_gen:
if not tg.path.is_child_of(ln):
continue
if Logs.verbose:
Logs.debug('group: %s'%tg)
tg.post()
def env_of_name(self,name):
try:
return self.all_envs[name]
except KeyError:
error('no such environment: '+name)
return None
def progress_line(self,state,total,col1,col2):
n=len(str(total))
Utils.rot_idx+=1
ind=Utils.rot_chr[Utils.rot_idx%4]
ini=self.ini
pc=(100.*state)/total
eta=Utils.get_elapsed_time(ini)
fs="[%%%dd/%%%dd][%%s%%2d%%%%%%s][%s]["%(n,n,ind)
left=fs%(state,total,col1,pc,col2)
right='][%s%s%s]'%(col1,eta,col2)
cols=Utils.get_term_cols()-len(left)-len(right)+2*len(col1)+2*len(col2)
if cols<7:cols=7
ratio=int((cols*state)/total)-1
bar=('='*ratio+'>').ljust(cols)
msg=Utils.indicator%(left,bar,right)
return msg
def do_install(self,src,tgt,chmod=O644):
if self.is_install>0:
if not Options.options.force:
try:
st1=os.stat(tgt)
st2=os.stat(src)
except OSError:
pass
else:
if st1.st_mtime>=st2.st_mtime and st1.st_size==st2.st_size:
return False
srclbl=src.replace(self.srcnode.abspath(None)+os.sep,'')
info("* installing %s as %s"%(srclbl,tgt))
try:os.remove(tgt)
except OSError:pass
try:
shutil.copy2(src,tgt)
os.chmod(tgt,chmod)
except IOError:
try:
os.stat(src)
except(OSError,IOError):
error('File %r does not exist'%src)
raise Utils.WafError('Could not install the file %r'%tgt)
return True
elif self.is_install<0:
info("* uninstalling %s"%tgt)
self.uninstall.append(tgt)
try:
os.remove(tgt)
except OSError,e:
if e.errno!=errno.ENOENT:
if not getattr(self,'uninstall_error',None):
self.uninstall_error=True
Logs.warn('build: some files could not be uninstalled (retry with -vv to list them)')
if Logs.verbose>1:
Logs.warn('could not remove %s (error code %r)'%(e.filename,e.errno))
return True
red=re.compile(r"^([A-Za-z]:)?[/\\\\]*")
def get_install_path(self,path,env=None):
if not env:env=self.env
destdir=env.get_destdir()
path=path.replace('/',os.sep)
destpath=Utils.subst_vars(path,env)
if destdir:
destpath=os.path.join(destdir,self.red.sub('',destpath))
return destpath
def install_dir(self,path,env=None):
if env:
assert isinstance(env,Environment.Environment),"invalid parameter"
else:
env=self.env
if not path:
return[]
destpath=self.get_install_path(path,env)
if self.is_install>0:
info('* creating %s'%destpath)
Utils.check_dir(destpath)
elif self.is_install<0:
info('* removing %s'%destpath)
self.uninstall.append(destpath+'/xxx')
def install_files(self,path,files,env=None,chmod=O644,relative_trick=False,cwd=None):
if env:
assert isinstance(env,Environment.Environment),"invalid parameter"
else:
env=self.env
if not path:return[]
if not cwd:
cwd=self.path
if isinstance(files,str)and'*'in files:
gl=cwd.abspath()+os.sep+files
lst=glob.glob(gl)
else:
lst=Utils.to_list(files)
if not getattr(lst,'__iter__',False):
lst=[lst]
destpath=self.get_install_path(path,env)
Utils.check_dir(destpath)
installed_files=[]
for filename in lst:
if isinstance(filename,str)and os.path.isabs(filename):
alst=Utils.split_path(filename)
destfile=os.path.join(destpath,alst[-1])
else:
if isinstance(filename,Node.Node):
nd=filename
else:
nd=cwd.find_resource(filename)
if not nd:
raise Utils.WafError("Unable to install the file %r (not found in %s)"%(filename,cwd))
if relative_trick:
destfile=os.path.join(destpath,filename)
Utils.check_dir(os.path.dirname(destfile))
else:
destfile=os.path.join(destpath,nd.name)
filename=nd.abspath(env)
if self.do_install(filename,destfile,chmod):
installed_files.append(destfile)
return installed_files
def install_as(self,path,srcfile,env=None,chmod=O644,cwd=None):
if env:
assert isinstance(env,Environment.Environment),"invalid parameter"
else:
env=self.env
if not path:
raise Utils.WafError("where do you want to install %r? (%r?)"%(srcfile,path))
if not cwd:
cwd=self.path
destpath=self.get_install_path(path,env)
dir,name=os.path.split(destpath)
Utils.check_dir(dir)
if isinstance(srcfile,Node.Node):
src=srcfile.abspath(env)
else:
src=srcfile
if not os.path.isabs(srcfile):
node=cwd.find_resource(srcfile)
if not node:
raise Utils.WafError("Unable to install the file %r (not found in %s)"%(srcfile,cwd))
src=node.abspath(env)
return self.do_install(src,destpath,chmod)
def symlink_as(self,path,src,env=None,cwd=None):
if sys.platform=='win32':
return
if not path:
raise Utils.WafError("where do you want to install %r? (%r?)"%(src,path))
tgt=self.get_install_path(path,env)
dir,name=os.path.split(tgt)
Utils.check_dir(dir)
if self.is_install>0:
link=False
if not os.path.islink(tgt):
link=True
elif os.readlink(tgt)!=src:
link=True
if link:
try:os.remove(tgt)
except OSError:pass
info('* symlink %s (-> %s)'%(tgt,src))
os.symlink(src,tgt)
return 0
else:
try:
info('* removing %s'%(tgt))
os.remove(tgt)
return 0
except OSError:
return 1
def exec_command(self,cmd,**kw):
debug('runner: system command -> %s',cmd)
if self.log:
self.log.write('%s\n'%cmd)
kw['log']=self.log
try:
if not kw.get('cwd',None):
kw['cwd']=self.cwd
except AttributeError:
self.cwd=kw['cwd']=self.bldnode.abspath()
return Utils.exec_command(cmd,**kw)
def printout(self,s):
f=self.log or sys.stderr
f.write(s)
f.flush()
def add_subdirs(self,dirs):
self.recurse(dirs,'build')
def pre_recurse(self,name_or_mod,path,nexdir):
if not hasattr(self,'oldpath'):
self.oldpath=[]
self.oldpath.append(self.path)
self.path=self.root.find_dir(nexdir)
return{'bld':self,'ctx':self}
def post_recurse(self,name_or_mod,path,nexdir):
self.path=self.oldpath.pop()
def pre_build(self):
if hasattr(self,'pre_funs'):
for m in self.pre_funs:
m(self)
def post_build(self):
if hasattr(self,'post_funs'):
for m in self.post_funs:
m(self)
def add_pre_fun(self,meth):
try:self.pre_funs.append(meth)
except AttributeError:self.pre_funs=[meth]
def add_post_fun(self,meth):
try:self.post_funs.append(meth)
except AttributeError:self.post_funs=[meth]
def use_the_magic(self):
Task.algotype=Task.MAXPARALLEL
Task.file_deps=Task.extract_deps
self.magic=True
install_as=group_method(install_as)
install_files=group_method(install_files)
symlink_as=group_method(symlink_as)