Update to waf 1.8.17.

.wafupdaterc

@@ -134,4 +134,5 @@ WAFLIB_STRIP_EXTRAS="
     why
     win32_opts
     xcode
+    xcode6
 "

waf

@@ -32,7 +32,7 @@ POSSIBILITY OF SUCH DAMAGE.
 
 import os, sys, inspect
 
-VERSION="1.8.14"
+VERSION="1.8.17"
 REVISION="x"
 GIT="x"
 INSTALL="x"

waflib/Build.py

@@ -44,6 +44,10 @@ POST_LAZY = 1
 POST_BOTH = 2
 """Post mode: post the task generators at once, then re-check them for each group"""
 
+PROTOCOL = -1
+if sys.platform == 'cli':
+	PROTOCOL = 0
+
 class BuildContext(Context.Context):
 	'''executes the build'''
 
@@ -322,7 +326,7 @@ class BuildContext(Context.Context):
 		try:
 			waflib.Node.pickle_lock.acquire()
 			waflib.Node.Nod3 = self.node_class
-			x = cPickle.dumps(data, -1)
+			x = cPickle.dumps(data, PROTOCOL)
 		finally:
 			waflib.Node.pickle_lock.release()
 
@@ -821,17 +825,10 @@ class inst(Task.Task):
 			else:
 				y = self.path.find_resource(x)
 				if not y:
-					if Logs.verbose:
-						Logs.warn('Could not find %s immediately (may cause broken builds)' % x)
-					idx = self.generator.bld.get_group_idx(self)
-					for tg in self.generator.bld.groups[idx]:
-						if not isinstance(tg, inst) and id(tg) != id(self):
-							tg.post()
-						y = self.path.find_resource(x)
-						if y:
-							break
+					if os.path.isabs(x):
+						y = self.bld.root.make_node(x)
 					else:
-						raise Errors.WafError('Could not find %r in %r' % (x, self.path))
+						y = self.path.make_node(x)
 			buf.append(y)
 		self.inputs = buf
 
@@ -1039,6 +1036,7 @@ class InstallContext(BuildContext):
 		:param postpone: execute the task immediately to perform the installation
 		:type postpone: bool
 		"""
+		assert(dest)
 		tsk = inst(env=env or self.env)
 		tsk.bld = self
 		tsk.path = cwd or self.path
@@ -1075,6 +1073,7 @@ class InstallContext(BuildContext):
 		:param postpone: execute the task immediately to perform the installation
 		:type postpone: bool
 		"""
+		assert(dest)
 		tsk = inst(env=env or self.env)
 		tsk.bld = self
 		tsk.path = cwd or self.path
@@ -1107,11 +1106,11 @@ class InstallContext(BuildContext):
 		:param relative_trick: make the symlink relative (default: ``False``)
 		:type relative_trick: bool
 		"""
-
 		if Utils.is_win32:
 			# symlinks *cannot* work on that platform
+			# TODO waf 1.9 - replace by install_as
 			return
-
+		assert(dest)
 		tsk = inst(env=env or self.env)
 		tsk.bld = self
 		tsk.dest = dest

waflib/Configure.py

@@ -151,14 +151,8 @@ class ConfigurationContext(Context.Context):
 			if ver:
 				app = "%s (%s)" % (app, ver)
 
-		now = time.ctime()
-		pyver = sys.hexversion
-		systype = sys.platform
-		args = " ".join(sys.argv)
-		wafver = Context.WAFVERSION
-		abi = Context.ABI
-		self.to_log(conf_template % vars())
-
+		params = {'now': time.ctime(), 'pyver': sys.hexversion, 'systype': sys.platform, 'args': " ".join(sys.argv), 'wafver': Context.WAFVERSION, 'abi': Context.ABI, 'app': app}
+		self.to_log(conf_template % params)
 		self.msg('Setting top to', self.srcnode.abspath())
 		self.msg('Setting out to', self.bldnode.abspath())
 
@@ -208,17 +202,17 @@ class ConfigurationContext(Context.Context):
 		"""
 		if not env.PREFIX:
 			if Options.options.prefix or Utils.is_win32:
-				env.PREFIX = os.path.abspath(os.path.expanduser(Options.options.prefix))
+				env.PREFIX = Utils.sane_path(Options.options.prefix)
 			else:
 				env.PREFIX = ''
 		if not env.BINDIR:
 			if Options.options.bindir:
-				env.BINDIR = os.path.abspath(os.path.expanduser(Options.options.bindir))
+				env.BINDIR = Utils.sane_path(Options.options.bindir)
 			else:
 				env.BINDIR = Utils.subst_vars('${PREFIX}/bin', env)
 		if not env.LIBDIR:
 			if Options.options.libdir:
-				env.LIBDIR = os.path.abspath(os.path.expanduser(Options.options.libdir))
+				env.LIBDIR = Utils.sane_path(Options.options.libdir)
 			else:
 				env.LIBDIR = Utils.subst_vars('${PREFIX}/lib%s' % Utils.lib64(), env)
 
@@ -347,6 +341,7 @@ def conf(f):
 			if mandatory:
 				raise
 
+	fun.__name__ = f.__name__
 	setattr(ConfigurationContext, f.__name__, fun)
 	setattr(Build.BuildContext, f.__name__, fun)
 	return f
@@ -422,7 +417,7 @@ def find_file(self, filename, path_list=[]):
 	"""
 	for n in Utils.to_list(filename):
 		for d in Utils.to_list(path_list):
-			p = os.path.join(d, n)
+			p = os.path.expanduser(os.path.join(d, n))
 			if os.path.exists(p):
 				return p
 	self.fatal('Could not find %r' % filename)
@@ -529,7 +524,6 @@ def find_binary(self, filenames, exts, paths):
 						return x
 	return None
 
-
 @conf
 def run_build(self, *k, **kw):
 	"""

waflib/Context.py

@@ -11,13 +11,13 @@ from waflib import Utils, Errors, Logs
 import waflib.Node
 
 # the following 3 constants are updated on each new release (do not touch)
-HEXVERSION=0x1080e00
+HEXVERSION=0x1081100
 """Constant updated on new releases"""
 
-WAFVERSION="1.8.14"
+WAFVERSION="1.8.17"
 """Constant updated on new releases"""
 
-WAFREVISION="ce8234c396bb246a20ea9f51594ee051d5b378e7"
+WAFREVISION="cd7579a727d1b390bf9cbf111c1b20e811370bc0"
 """Git revision when the waf version is updated"""
 
 ABI = 98
@@ -307,6 +307,10 @@ class Context(ctx):
 				elif not node:
 					if not mandatory:
 						continue
+					try:
+						os.listdir(d)
+					except OSError:
+						raise Errors.WafError('Cannot read the folder %r' % d)
 					raise Errors.WafError('No wscript file in directory %s' % d)
 
 	def exec_command(self, cmd, **kw):
@@ -610,7 +614,7 @@ class Context(ctx):
 				doban = False
 				for b in ban:
 					r = b.replace("*", ".*")
-					if re.match(b, f):
+					if re.match(r, f):
 						doban = True
 				if not doban:
 					f = f.replace('.py', '')

waflib/Logs.py

@@ -6,7 +6,7 @@
 logging, colors, terminal width and pretty-print
 """
 
-import os, re, traceback, sys, types
+import os, re, traceback, sys
 from waflib import Utils, ansiterm
 
 if not os.environ.get('NOSYNC', False):
@@ -43,6 +43,11 @@ colors_lst = {
 
 indicator = '\r\x1b[K%s%s%s'
 
+try:
+	unicode
+except NameError:
+	unicode = None
+
 def enable_colors(use):
 	if use == 1:
 		if not (sys.stderr.isatty() or sys.stdout.isatty()):
@@ -150,7 +155,7 @@ class log_handler(logging.StreamHandler):
 	def emit_override(self, record, **kw):
 		self.terminator = getattr(record, 'terminator', '\n')
 		stream = self.stream
-		if hasattr(types, "UnicodeType"):
+		if unicode:
 			# python2
 			msg = self.formatter.format(record)
 			fs = '%s' + self.terminator
@@ -316,7 +321,7 @@ def free_logger(logger):
 		for x in logger.handlers:
 			x.close()
 			logger.removeHandler(x)
-	except Exception as e:
+	except Exception:
 		pass
 
 def pprint(col, msg, label='', sep='\n'):

waflib/Node.py

@@ -59,14 +59,11 @@ Ant patterns for files and folders to exclude while doing the
 recursive traversal in :py:meth:`waflib.Node.Node.ant_glob`
 """
 
-# TODO waf 1.9
-split_path = Utils.split_path_unix
+# TODO remove in waf 1.9
+split_path = Utils.split_path
+split_path_unix = Utils.split_path_unix
 split_path_cygwin = Utils.split_path_cygwin
 split_path_win32 = Utils.split_path_win32
-if sys.platform == 'cygwin':
-	split_path = split_path_cygwin
-elif Utils.is_win32:
-	split_path = split_path_win32
 
 class Node(object):
 	"""
@@ -152,6 +149,69 @@ class Node(object):
 		"""
 		Utils.writef(self.abspath(), data, flags, encoding)
 
+	def read_json(self, convert=True, encoding='utf-8'):
+		"""
+		Read and parse the contents of this node as JSON::
+
+			def build(bld):
+				bld.path.find_node('abc.json').read_json()
+
+		Note that this by default automatically decodes unicode strings on Python2, unlike what the Python JSON module does.
+
+		:type  convert: boolean
+		:param convert: Prevents decoding of unicode strings on Python2
+		:type  encoding: string
+		:param encoding: The encoding of the file to read. This default to UTF8 as per the JSON standard
+		:rtype: object
+		:return: Parsed file contents
+		"""
+		import json # Python 2.6 and up
+		object_pairs_hook = None
+		if convert and sys.hexversion < 0x3000000:
+			try:
+				_type = unicode
+			except NameError:
+				_type = str
+
+			def convert(value):
+				if isinstance(value, list):
+					return [convert(element) for element in value]
+				elif isinstance(value, _type):
+					return str(value)
+				else:
+					return value
+
+			def object_pairs(pairs):
+				return dict((str(pair[0]), convert(pair[1])) for pair in pairs)
+
+			object_pairs_hook = object_pairs
+
+		return json.loads(self.read(encoding=encoding), object_pairs_hook=object_pairs_hook)
+
+	def write_json(self, data, pretty=True):
+		"""
+		Writes a python object as JSON to disk. Files are always written as UTF8 as per the JSON standard::
+
+			def build(bld):
+				bld.path.find_node('xyz.json').write_json(199)
+
+		:type  data: object
+		:param data: The data to write to disk
+		:type  pretty: boolean
+		:param pretty: Determines if the JSON will be nicely space separated
+		"""
+		import json # Python 2.6 and up
+		indent = 2
+		separators = (',', ': ')
+		sort_keys = pretty
+		newline = os.linesep
+		if not pretty:
+			indent = None
+			separators = (',', ':')
+			newline = ''
+		output = json.dumps(data, indent=indent, separators=separators, sort_keys=sort_keys) + newline
+		self.write(output, encoding='utf-8')
+
 	def chmod(self, val):
 		"""
 		Change file/dir permissions::
@@ -251,7 +311,7 @@ class Node(object):
 				cur.children = self.dict_class()
 			else:
 				try:
-					cur = cur.children[x]
+					cur = ch[x]
 					continue
 				except KeyError:
 					pass

waflib/Options.py

@@ -256,7 +256,7 @@ class OptionsContext(Context.Context):
 				commands.append(arg)
 
 		if options.destdir:
-			options.destdir = os.path.abspath(os.path.expanduser(options.destdir))
+			options.destdir = Utils.sane_path(options.destdir)
 
 		if options.verbose >= 1:
 			self.load('errcheck')

waflib/Runner.py

@@ -278,6 +278,13 @@ class Parallel(object):
 		:param tsk: task
 		:type tsk: :py:attr:`waflib.Task.TaskBase`
 		"""
+		if hasattr(tsk, 'scan') and hasattr(tsk, 'uid'):
+			# TODO waf 1.9 - this breaks encapsulation
+			key = (tsk.uid(), 'imp')
+			try:
+				del self.bld.task_sigs[key]
+			except KeyError:
+				pass
 		if not self.bld.keep:
 			self.stop = True
 		self.error.append(tsk)

waflib/Scripting.py

@@ -46,6 +46,7 @@ def waf_entry_point(current_directory, version, wafdir):
 		# perhaps extract 'wscript' as a constant
 		if os.path.basename(potential_wscript) == 'wscript' and os.path.isfile(potential_wscript):
 			# need to explicitly normalize the path, as it may contain extra '/.'
+			# TODO abspath?
 			current_directory = os.path.normpath(os.path.dirname(potential_wscript))
 			sys.argv.pop(1)
 
@@ -62,11 +63,14 @@ def waf_entry_point(current_directory, version, wafdir):
 					break
 
 	# if --top is provided assume the build started in the top directory
-	for x in sys.argv:
+	for i, x in enumerate(sys.argv):
+		# WARNING: this modifies sys.argv
 		if x.startswith('--top='):
-			Context.run_dir = Context.top_dir = x[6:]
+			Context.run_dir = Context.top_dir = Utils.sane_path(x[6:])
+			sys.argv[i] = '--top=' + Context.run_dir
 		if x.startswith('--out='):
-			Context.out_dir = x[6:]
+			Context.out_dir = Utils.sane_path(x[6:])
+			sys.argv[i] = '--out=' + Context.out_dir
 
 	# try to find a lock file (if the project was configured)
 	# at the same time, store the first wscript file seen
@@ -137,7 +141,7 @@ def waf_entry_point(current_directory, version, wafdir):
 		sys.exit(1)
 
 	try:
-		set_main_module(os.path.join(Context.run_dir, Context.WSCRIPT_FILE))
+		set_main_module(os.path.normpath(os.path.join(Context.run_dir, Context.WSCRIPT_FILE)))
 	except Errors.WafError as e:
 		Logs.pprint('RED', e.verbose_msg)
 		Logs.error(str(e))

waflib/Task.py

@@ -37,11 +37,14 @@ SKIP_ME = -2
 RUN_ME = -3
 """The task must be executed"""
 
+# To save some memory during the build, consider discarding tsk.last_cmd in the two templates below
+
 COMPILE_TEMPLATE_SHELL = '''
 def f(tsk):
 	env = tsk.env
 	gen = tsk.generator
 	bld = gen.bld
+	cwdx = getattr(bld, 'cwdx', bld.bldnode) # TODO single cwd value in waf 1.9
 	wd = getattr(tsk, 'cwd', None)
 	p = env.get_flat
 	tsk.last_cmd = cmd = \'\'\' %s \'\'\' % s
@@ -53,6 +56,7 @@ def f(tsk):
 	env = tsk.env
 	gen = tsk.generator
 	bld = gen.bld
+	cwdx = getattr(bld, 'cwdx', bld.bldnode) # TODO single cwd value in waf 1.9
 	wd = getattr(tsk, 'cwd', None)
 	def to_list(xx):
 		if isinstance(xx, str): return [xx]
@@ -81,11 +85,10 @@ class store_task_type(type):
 			name = name.replace('_task', '')
 		if name != 'evil' and name != 'TaskBase':
 			global classes
-
 			if getattr(cls, 'run_str', None):
 				# if a string is provided, convert it to a method
 				(f, dvars) = compile_fun(cls.run_str, cls.shell)
-				cls.hcode = cls.run_str
+				cls.hcode = Utils.h_cmd(cls.run_str)
 				cls.orig_run_str = cls.run_str
 				# change the name of run_str or it is impossible to subclass with a function
 				cls.run_str = None
@@ -94,10 +97,7 @@ class store_task_type(type):
 				cls.vars.sort()
 			elif getattr(cls, 'run', None) and not 'hcode' in cls.__dict__:
 				# getattr(cls, 'hcode') would look in the upper classes
-				cls.hcode = Utils.h_fun(cls.run)
-
-			if sys.hexversion > 0x3000000:
-				cls.hcode = cls.hcode.encode('iso8859-1', 'xmlcharrefreplace')
+				cls.hcode = Utils.h_cmd(cls.run)
 
 			# be creative
 			getattr(cls, 'register', classes)[name] = cls
@@ -206,6 +206,7 @@ class TaskBase(evil):
 		# remove the task signature immediately before it is executed
 		# in case of failure the task will be executed again
 		try:
+			# TODO waf 1.9 - this breaks encapsulation
 			del self.generator.bld.task_sigs[self.uid()]
 		except KeyError:
 			pass
@@ -972,17 +973,17 @@ def compile_fun_shell(line):
 	for (var, meth) in extr:
 		if var == 'SRC':
 			if meth: app('tsk.inputs%s' % meth)
-			else: app('" ".join([a.path_from(bld.bldnode) for a in tsk.inputs])')
+			else: app('" ".join([a.path_from(cwdx) for a in tsk.inputs])')
 		elif var == 'TGT':
 			if meth: app('tsk.outputs%s' % meth)
-			else: app('" ".join([a.path_from(bld.bldnode) for a in tsk.outputs])')
+			else: app('" ".join([a.path_from(cwdx) for a in tsk.outputs])')
 		elif meth:
 			if meth.startswith(':'):
 				m = meth[1:]
 				if m == 'SRC':
-					m = '[a.path_from(bld.bldnode) for a in tsk.inputs]'
+					m = '[a.path_from(cwdx) for a in tsk.inputs]'
 				elif m == 'TGT':
-					m = '[a.path_from(bld.bldnode) for a in tsk.outputs]'
+					m = '[a.path_from(cwdx) for a in tsk.outputs]'
 				elif m[:3] not in ('tsk', 'gen', 'bld'):
 					dvars.extend([var, meth[1:]])
 					m = '%r' % m
@@ -1027,17 +1028,17 @@ def compile_fun_noshell(line):
 		(var, meth) = extr[x]
 		if var == 'SRC':
 			if meth: app('lst.append(tsk.inputs%s)' % meth)
-			else: app("lst.extend([a.path_from(bld.bldnode) for a in tsk.inputs])")
+			else: app("lst.extend([a.path_from(cwdx) for a in tsk.inputs])")
 		elif var == 'TGT':
 			if meth: app('lst.append(tsk.outputs%s)' % meth)
-			else: app("lst.extend([a.path_from(bld.bldnode) for a in tsk.outputs])")
+			else: app("lst.extend([a.path_from(cwdx) for a in tsk.outputs])")
 		elif meth:
 			if meth.startswith(':'):
 				m = meth[1:]
 				if m == 'SRC':
-					m = '[a.path_from(bld.bldnode) for a in tsk.inputs]'
+					m = '[a.path_from(cwdx) for a in tsk.inputs]'
 				elif m == 'TGT':
-					m = '[a.path_from(bld.bldnode) for a in tsk.outputs]'
+					m = '[a.path_from(cwdx) for a in tsk.outputs]'
 				elif m[:3] not in ('tsk', 'gen', 'bld'):
 					dvars.extend([var, m])
 					m = '%r' % m
@@ -1074,9 +1075,27 @@ def compile_fun(line, shell=False):
 	The reserved keywords *TGT* and *SRC* represent the task input and output nodes
 
 	"""
-	if line.find('<') > 0 or line.find('>') > 0 or line.find('&&') > 0:
-		shell = True
-
+	if isinstance(line, str):
+		if line.find('<') > 0 or line.find('>') > 0 or line.find('&&') > 0:
+			shell = True
+	else:
+		dvars_lst = []
+		funs_lst = []
+		for x in line:
+			if isinstance(x, str):
+				fun, dvars = compile_fun(x, shell)
+				dvars_lst += dvars
+				funs_lst.append(fun)
+			else:
+				# assume a function to let through
+				funs_lst.append(x)
+		def composed_fun(task):
+			for x in funs_lst:
+				ret = x(task)
+				if ret:
+					return ret
+			return None
+		return composed_fun, dvars
 	if shell:
 		return compile_fun_shell(line)
 	else:
@@ -1111,7 +1130,7 @@ def task_factory(name, func=None, vars=None, color='GREEN', ext_in=[], ext_out=[
 		'scan': scan,
 	}
 
-	if isinstance(func, str):
+	if isinstance(func, str) or isinstance(func, tuple):
 		params['run_str'] = func
 	else:
 		params['run'] = func

waflib/TaskGen.py

@@ -564,7 +564,15 @@ def process_rule(self):
 		except KeyError:
 			pass
 	if not cls:
-		cls = Task.task_factory(name, self.rule,
+
+		rule = self.rule
+		if hasattr(self, 'chmod'):
+			def chmod_fun(tsk):
+				for x in tsk.outputs:
+					os.chmod(x.abspath(), self.chmod)
+			rule = (self.rule, chmod_fun)
+
+		cls = Task.task_factory(name, rule,
 			getattr(self, 'vars', []),
 			shell=getattr(self, 'shell', True), color=getattr(self, 'color', 'BLUE'),
 			scan = getattr(self, 'scan', None))
@@ -614,7 +622,7 @@ def process_rule(self):
 				x.parent.mkdir() # if a node was given, create the required folders
 				tsk.outputs.append(x)
 		if getattr(self, 'install_path', None):
-			self.bld.install_files(self.install_path, tsk.outputs)
+			self.bld.install_files(self.install_path, tsk.outputs, chmod=getattr(self, 'chmod', Utils.O644))
 
 	if getattr(self, 'source', None):
 		tsk.inputs = self.to_nodes(self.source)
@@ -669,24 +677,34 @@ class subst_pc(Task.Task):
 	in the substitution changes.
 	"""
 
+	def force_permissions(self):
+		"Private for the time being, we will probably refactor this into run_str=[run1,chmod]"
+		if getattr(self.generator, 'chmod', None):
+			for x in self.outputs:
+				os.chmod(x.abspath(), self.generator.chmod)
+
 	def run(self):
 		"Substitutes variables in a .in file"
 
 		if getattr(self.generator, 'is_copy', None):
-			self.outputs[0].write(self.inputs[0].read('rb'), 'wb')
-			if getattr(self.generator, 'chmod', None):
-				os.chmod(self.outputs[0].abspath(), self.generator.chmod)
+			for i, x in enumerate(self.outputs):
+				x.write(self.inputs[i].read('rb'), 'wb')
+			self.force_permissions()
 			return None
 
 		if getattr(self.generator, 'fun', None):
-			return self.generator.fun(self)
+			ret = self.generator.fun(self)
+			if not ret:
+				self.force_permissions()
+			return ret
 
 		code = self.inputs[0].read(encoding=getattr(self.generator, 'encoding', 'ISO8859-1'))
 		if getattr(self.generator, 'subst_fun', None):
 			code = self.generator.subst_fun(self, code)
 			if code is not None:
 				self.outputs[0].write(code, encoding=getattr(self.generator, 'encoding', 'ISO8859-1'))
-			return
+			self.force_permissions()
+			return None
 
 		# replace all % by %% to prevent errors by % signs
 		code = code.replace('%', '%%')
@@ -722,8 +740,7 @@ class subst_pc(Task.Task):
 		try: delattr(self, 'cache_sig')
 		except AttributeError: pass
 
-		if getattr(self.generator, 'chmod', None):
-			os.chmod(self.outputs[0].abspath(), self.generator.chmod)
+		self.force_permissions()
 
 	def sig_vars(self):
 		"""
@@ -816,7 +833,7 @@ def process_subst(self):
 				b = y
 
 		if not a:
-			raise Errors.WafError('cound not find %r for %r' % (x, self))
+			raise Errors.WafError('could not find %r for %r' % (x, self))
 
 		has_constraints = False
 		tsk = self.create_task('subst', a, b)

waflib/Tools/c_aliases.py

@@ -4,8 +4,7 @@
 
 "base for all c/c++ programs and libraries"
 
-import os, sys, re
-from waflib import Utils, Build, Errors
+from waflib import Utils, Errors
 from waflib.Configure import conf
 
 def get_extensions(lst):
@@ -49,7 +48,7 @@ def sniff_features(**kw):
 			feats.append('cxx')
 			break
 
-	if 'c' in exts or 'vala' in exts:
+	if 'c' in exts or 'vala' in exts or 'gs' in exts:
 		feats.append('c')
 
 	for x in 'f f90 F F90 for FOR'.split():
@@ -71,7 +70,7 @@ def sniff_features(**kw):
 				feats.append(x + type)
 				will_link = True
 		if not will_link and not kw.get('features', []):
-			raise Errors.WafError('Cannot link from %r, try passing eg: features="cprogram"?' % kw)
+			raise Errors.WafError('Cannot link from %r, try passing eg: features="c cprogram"?' % kw)
 	return feats
 
 def set_features(kw, _type):

waflib/Tools/c_config.py

@@ -6,7 +6,7 @@
 C/C++/D configuration helpers
 """
 
-import os, re, shlex, sys
+import os, re, shlex
 from waflib import Build, Utils, Task, Options, Logs, Errors, Runner
 from waflib.TaskGen import after_method, feature
 from waflib.Configure import conf
@@ -288,11 +288,11 @@ def exec_cfg(self, kw):
 	"""
 
 	path = Utils.to_list(kw['path'])
-
+	env = self.env.env or None
 	def define_it():
 		pkgname = kw.get('uselib_store', kw['package'].upper())
 		if kw.get('global_define'):
-			# compatibility
+			# compatibility, replace by pkgname in WAF 1.9?
 			self.define(self.have_define(kw['package']), 1, False)
 		else:
 			self.env.append_unique('DEFINES_%s' % pkgname, "%s=1" % self.have_define(pkgname))
@@ -301,7 +301,7 @@ def exec_cfg(self, kw):
 	# pkg-config version
 	if 'atleast_pkgconfig_version' in kw:
 		cmd = path + ['--atleast-pkgconfig-version=%s' % kw['atleast_pkgconfig_version']]
-		self.cmd_and_log(cmd)
+		self.cmd_and_log(cmd, env=env)
 		if not 'okmsg' in kw:
 			kw['okmsg'] = 'yes'
 		return
@@ -310,7 +310,7 @@ def exec_cfg(self, kw):
 	for x in cfg_ver:
 		y = x.replace('-', '_')
 		if y in kw:
-			self.cmd_and_log(path + ['--%s=%s' % (x, kw[y]), kw['package']])
+			self.cmd_and_log(path + ['--%s=%s' % (x, kw[y]), kw['package']], env=env)
 			if not 'okmsg' in kw:
 				kw['okmsg'] = 'yes'
 			define_it()
@@ -318,7 +318,7 @@ def exec_cfg(self, kw):
 
 	# retrieving the version of a module
 	if 'modversion' in kw:
-		version = self.cmd_and_log(path + ['--modversion', kw['modversion']]).strip()
+		version = self.cmd_and_log(path + ['--modversion', kw['modversion']], env=env).strip()
 		self.define('%s_VERSION' % Utils.quote_define_name(kw.get('uselib_store', kw['modversion'])), version)
 		return version
 
@@ -342,19 +342,19 @@ def exec_cfg(self, kw):
 
 	# retrieving variables of a module
 	if 'variables' in kw:
-		env = kw.get('env', self.env)
+		v = kw.get('env', self.env)
 		uselib = kw.get('uselib_store', kw['package'].upper())
 		vars = Utils.to_list(kw['variables'])
 		for v in vars:
-			val = self.cmd_and_log(lst + ['--variable=' + v]).strip()
+			val = self.cmd_and_log(lst + ['--variable=' + v], env=env).strip()
 			var = '%s_%s' % (uselib, v)
-			env[var] = val
+			v[var] = val
 		if not 'okmsg' in kw:
 			kw['okmsg'] = 'yes'
 		return
 
 	# so we assume the command-line will output flags to be parsed afterwards
-	ret = self.cmd_and_log(lst)
+	ret = self.cmd_and_log(lst, env=env)
 	if not 'okmsg' in kw:
 		kw['okmsg'] = 'yes'
 
@@ -482,7 +482,10 @@ def validate_c(self, kw):
 		kw['type'] = 'cprogram'
 
 	if not 'features' in kw:
-		kw['features'] = [kw['compile_mode'], kw['type']] # "cprogram c"
+		if not 'header_name' in kw or kw.get('link_header_test', True):
+			kw['features'] = [kw['compile_mode'], kw['type']] # "c ccprogram"
+		else:
+			kw['features'] = [kw['compile_mode']]
 	else:
 		kw['features'] = Utils.to_list(kw['features'])
 
@@ -604,6 +607,11 @@ def validate_c(self, kw):
 	if self.env[INCKEYS]:
 		kw['code'] = '\n'.join(['#include <%s>' % x for x in self.env[INCKEYS]]) + '\n' + kw['code']
 
+	# in case defines lead to very long command-lines
+	if kw.get('merge_config_header', False) or env.merge_config_header:
+		kw['code'] = '%s\n\n%s' % (self.get_config_header(), kw['code'])
+		env.DEFINES = [] # modify the copy
+
 	if not kw.get('success'): kw['success'] = None
 
 	if 'define_name' in kw:
@@ -626,7 +634,7 @@ def post_check(self, *k, **kw):
 		is_success = (kw['success'] == 0)
 
 	if 'define_name' in kw:
-		# TODO simplify?
+		# TODO simplify!
 		if 'header_name' in kw or 'function_name' in kw or 'type_name' in kw or 'fragment' in kw:
 			if kw['execute'] and kw.get('define_ret', None) and isinstance(is_success, str):
 				self.define(kw['define_name'], is_success, quote=kw.get('quote', 1))
@@ -635,6 +643,10 @@ def post_check(self, *k, **kw):
 		else:
 			self.define_cond(kw['define_name'], is_success)
 
+		# consistency with check_cfg
+		if kw.get('global_define', None):
+			self.env[kw['define_name']] = is_success
+
 	if 'header_name' in kw:
 		if kw.get('auto_add_header_name', False):
 			self.env.append_value(INCKEYS, Utils.to_list(kw['header_name']))

waflib/Tools/c_osx.py

@@ -6,8 +6,8 @@
 MacOSX related tools
 """
 
-import os, shutil, sys, platform
-from waflib import TaskGen, Task, Build, Options, Utils, Errors
+import os, shutil, platform
+from waflib import Task, Utils, Errors
 from waflib.TaskGen import taskgen_method, feature, after_method, before_method
 
 app_info = '''
@@ -48,7 +48,6 @@ def create_bundle_dirs(self, name, out):
 	"""
 	Create bundle folders, used by :py:func:`create_task_macplist` and :py:func:`create_task_macapp`
 	"""
-	bld = self.bld
 	dir = out.parent.find_or_declare(name)
 	dir.mkdir()
 	macos = dir.find_or_declare(['Contents', 'MacOS'])
@@ -102,7 +101,7 @@ def create_task_macapp(self):
 			inst_to = getattr(self, 'install_path', '/Applications') + '/%s/Resources' % name
 			for node in self.to_nodes(self.mac_files):
 				relpath = node.path_from(mac_files_root or node.parent)
-				tsk = self.create_task('macapp', node, res_dir.make_node(relpath))
+				self.create_task('macapp', node, res_dir.make_node(relpath))
 				self.bld.install_as(os.path.join(inst_to, relpath), node)
 
 		if getattr(self, 'mac_resources', None):
@@ -121,7 +120,7 @@ def create_task_macapp(self):
 					nodes = [node]
 				for node in nodes:
 					rel = node.path_from(parent)
-					tsk = self.create_task('macapp', node, res_dir.make_node(rel))
+					self.create_task('macapp', node, res_dir.make_node(rel))
 					self.bld.install_as(inst_to + '/%s' % rel, node)
 
 		if getattr(self.bld, 'is_install', None):

waflib/Tools/c_tests.py

@@ -9,7 +9,6 @@ Various configuration tests.
 from waflib import Task
 from waflib.Configure import conf
 from waflib.TaskGen import feature, before_method, after_method
-import sys
 
 LIB_CODE = '''
 #ifdef _MSC_VER

waflib/Tools/ccroot.py

@@ -25,8 +25,8 @@ USELIB_VARS['cxx']      = set(['INCLUDES', 'FRAMEWORKPATH', 'DEFINES', 'CPPFLAGS
 USELIB_VARS['d']        = set(['INCLUDES', 'DFLAGS'])
 USELIB_VARS['includes'] = set(['INCLUDES', 'FRAMEWORKPATH', 'ARCH'])
 
-USELIB_VARS['cprogram'] = USELIB_VARS['cxxprogram'] = set(['LIB', 'STLIB', 'LIBPATH', 'STLIBPATH', 'LINKFLAGS', 'RPATH', 'LINKDEPS', 'FRAMEWORK', 'FRAMEWORKPATH', 'ARCH'])
-USELIB_VARS['cshlib']   = USELIB_VARS['cxxshlib']   = set(['LIB', 'STLIB', 'LIBPATH', 'STLIBPATH', 'LINKFLAGS', 'RPATH', 'LINKDEPS', 'FRAMEWORK', 'FRAMEWORKPATH', 'ARCH'])
+USELIB_VARS['cprogram'] = USELIB_VARS['cxxprogram'] = set(['LIB', 'STLIB', 'LIBPATH', 'STLIBPATH', 'LINKFLAGS', 'RPATH', 'LINKDEPS', 'FRAMEWORK', 'FRAMEWORKPATH', 'ARCH', 'LDFLAGS'])
+USELIB_VARS['cshlib']   = USELIB_VARS['cxxshlib']   = set(['LIB', 'STLIB', 'LIBPATH', 'STLIBPATH', 'LINKFLAGS', 'RPATH', 'LINKDEPS', 'FRAMEWORK', 'FRAMEWORKPATH', 'ARCH', 'LDFLAGS'])
 USELIB_VARS['cstlib']   = USELIB_VARS['cxxstlib']   = set(['ARFLAGS', 'LINKDEPS'])
 
 USELIB_VARS['dprogram'] = set(['LIB', 'STLIB', 'LIBPATH', 'STLIBPATH', 'LINKFLAGS', 'RPATH', 'LINKDEPS'])
@@ -158,7 +158,10 @@ class link_task(Task.Task):
 					if len(nums) >= 2:
 						pattern += '.%s' % nums[1]
 
-			tmp = folder + os.sep + pattern % name
+			if folder:
+				tmp = folder + os.sep + pattern % name
+			else:
+				tmp = pattern % name
 			target = self.generator.path.find_or_declare(tmp)
 		self.set_outputs(target)
 

waflib/Tools/clang.py

@@ -6,7 +6,6 @@
 Detect the Clang C compiler
 """
 
-import os, sys
 from waflib.Tools import ccroot, ar, gcc
 from waflib.Configure import conf
 

waflib/Tools/clangxx.py

@@ -6,7 +6,6 @@
 Detect the Clang++ C++ compiler
 """
 
-import os, sys
 from waflib.Tools import ccroot, ar, gxx
 from waflib.Configure import conf
 

waflib/Tools/compiler_c.py

@@ -30,9 +30,9 @@ Not all compilers need to have a specific tool. For example, the clang compilers
 	$ CC=clang waf configure
 """
 
-import os, sys, imp, types, re
+import re
 from waflib.Tools import ccroot
-from waflib import Utils, Configure
+from waflib import Utils
 from waflib.Logs import debug
 
 c_compiler = {

waflib/Tools/compiler_cxx.py

@@ -31,9 +31,9 @@ Not all compilers need to have a specific tool. For example, the clang compilers
 """
 
 
-import os, sys, imp, types, re
+import re
 from waflib.Tools import ccroot
-from waflib import Utils, Configure
+from waflib import Utils
 from waflib.Logs import debug
 
 cxx_compiler = {

waflib/Tools/errcheck.py

@@ -22,6 +22,7 @@ typos = {
 
 meths_typos = ['__call__', 'program', 'shlib', 'stlib', 'objects']
 
+import sys
 from waflib import Logs, Build, Node, Task, TaskGen, ConfigSet, Errors, Utils
 import waflib.Tools.ccroot
 
@@ -84,6 +85,9 @@ def check_invalid_constraints(self):
 
 	# the build scripts have been read, so we can check for invalid after/before attributes on task classes
 	for cls in list(Task.classes.values()):
+		if sys.hexversion > 0x3000000 and issubclass(cls, Task.Task) and isinstance(cls.hcode, str):
+			raise Errors.WafError('Class %r has hcode value %r of type <str>, expecting <bytes> (use Utils.h_cmd() ?)' % (cls, cls.hcode))
+
 		for x in ('before', 'after'):
 			for y in Utils.to_list(getattr(cls, x, [])):
 				if not Task.classes.get(y, None):
@@ -103,7 +107,6 @@ def replace(m):
 			if x in kw:
 				if x == 'iscopy' and 'subst' in getattr(self, 'features', ''):
 					continue
-				err = True
 				Logs.error('Fix the typo %r -> %r on %r' % (x, typos[x], ret))
 		return ret
 	setattr(Build.BuildContext, m, call)

waflib/Tools/gcc.py

@@ -8,8 +8,6 @@
 gcc/llvm detection.
 """
 
-import os, sys
-from waflib import Configure, Options, Utils
 from waflib.Tools import ccroot, ar
 from waflib.Configure import conf
 

waflib/Tools/gxx.py

@@ -8,8 +8,6 @@
 g++/llvm detection.
 """
 
-import os, sys
-from waflib import Configure, Options, Utils
 from waflib.Tools import ccroot, ar
 from waflib.Configure import conf
 

waflib/Tools/icc.py

@@ -7,7 +7,7 @@
 Detect the Intel C compiler
 """
 
-import os, sys
+import sys
 from waflib.Tools import ccroot, ar, gcc
 from waflib.Configure import conf
 

waflib/Tools/icpc.py

@@ -6,7 +6,7 @@
 Detect the Intel C++ compiler
 """
 
-import os, sys
+import sys
 from waflib.Tools import ccroot, ar, gxx
 from waflib.Configure import conf
 

waflib/Tools/irixcc.py

@@ -6,8 +6,6 @@ compiler definition for irix/MIPSpro cc compiler
 based on suncc.py from waf
 """
 
-import os
-from waflib import Utils
 from waflib.Tools import ccroot, ar
 from waflib.Configure import conf
 

waflib/Tools/msvc.py

@@ -194,10 +194,14 @@ echo LIB=%%LIB%%;%%LIBPATH%%
 	try:
 		try:
 			conf.cmd_and_log(cxx + ['/help'], env=env)
+		except UnicodeError:
+			st = Utils.ex_stack()
+			if conf.logger:
+				conf.logger.error(st)
+			conf.fatal('msvc: Unicode error - check the code page?')
 		except Exception as e:
-			debug('msvc: get_msvc_version: %r %r %r -> failure' % (compiler, version, target))
-			debug(str(e))
-			conf.fatal('msvc: cannot run the compiler (in get_msvc_version)')
+			debug('msvc: get_msvc_version: %r %r %r -> failure %s' % (compiler, version, target, str(e)))
+			conf.fatal('msvc: cannot run the compiler in get_msvc_version (run with -v to display errors)')
 		else:
 			debug('msvc: get_msvc_version: %r %r %r -> OK', compiler, version, target)
 	finally:
@@ -235,7 +239,7 @@ def gather_wsdk_versions(conf, versions):
 			path,type = Utils.winreg.QueryValueEx(msvc_version,'InstallationFolder')
 		except WindowsError:
 			continue
-		if os.path.isfile(os.path.join(path, 'bin', 'SetEnv.cmd')):
+		if path and os.path.isfile(os.path.join(path, 'bin', 'SetEnv.cmd')):
 			targets = []
 			for target,arch in all_msvc_platforms:
 				try:
@@ -328,8 +332,8 @@ def get_compiler_env(conf, compiler, version, bat_target, bat, select=None):
 	"""
 	Gets the compiler environment variables as a tuple. Evaluation is eager by default.
 	If set to lazy with ``--msvc_lazy_autodetect`` or ``env.MSVC_LAZY_AUTODETECT``
-        the environment is evaluated when the tuple is destructured or iterated. This means
-        destructuring can throw :py:class:`conf.errors.ConfigurationError`.
+	the environment is evaluated when the tuple is destructured or iterated. This means
+	destructuring can throw :py:class:`conf.errors.ConfigurationError`.
 
 	:param conf: configuration context to use to eventually get the version environment
 	:param compiler: compiler name
@@ -350,26 +354,26 @@ def get_compiler_env(conf, compiler, version, bat_target, bat, select=None):
 class lazytup(object):
 	"""
 	A tuple that evaluates its elements from a function when iterated or destructured.
-	
+
 	:param fn: thunk to evaluate the tuple on demand
 	:param lazy: whether to delay evaluation or evaluate in the constructor
 	:param default: optional default for :py:func:`repr` if it should not evaluate
 	"""
-        def __init__(self, fn, lazy=True, default=None):
+	def __init__(self, fn, lazy=True, default=None):
 		self.fn = fn
 		self.default = default
 		if not lazy:
 			self.evaluate()
-        def __len__(self):
+	def __len__(self):
 		self.evaluate()
-                return len(self.value)
-        def __iter__(self):
+		return len(self.value)
+	def __iter__(self):
 		self.evaluate()
-                for i, v in enumerate(self.value):
-                        yield v
-        def __getitem__(self, i):
+		for i, v in enumerate(self.value):
+			yield v
+	def __getitem__(self, i):
 		self.evaluate()
-                return self.value[i]
+		return self.value[i]
 	def __repr__(self):
 		if hasattr(self, 'value'):
 			return repr(self.value)
@@ -594,9 +598,9 @@ def gather_intel_composer_versions(conf, versions):
 							if (r'if exist "%VS110COMNTOOLS%..\IDE\VSWinExpress.exe"' in Utils.readf(compilervars_arch) and
 								not os.path.exists(vs_express_path) and not os.path.exists(dev_env_path)):
 								Logs.warn(('The Intel compilervar_arch.bat only checks for one Visual Studio SKU '
-								           '(VSWinExpress.exe) but it does not seem to be installed at %r. '
-								           'The intel command line set up will fail to configure unless the file %r'
-								           'is patched. See: %s') % (vs_express_path, compilervars_arch, patch_url))
+								'(VSWinExpress.exe) but it does not seem to be installed at %r. '
+								'The intel command line set up will fail to configure unless the file %r'
+								'is patched. See: %s') % (vs_express_path, compilervars_arch, patch_url))
 			except WindowsError:
 				pass
 		major = version[0:2]
@@ -833,9 +837,6 @@ def find_msvc(conf):
 	v.MSVC_MANIFEST = (compiler == 'msvc' and version >= 8) or (compiler == 'wsdk' and version >= 6) or (compiler == 'intel' and version >= 11)
 
 	# compiler
-	cxx = None
-	if v['CXX']: cxx = v['CXX']
-	elif 'CXX' in conf.environ: cxx = conf.environ['CXX']
 	cxx = conf.find_program(compiler_name, var='CXX', path_list=path)
 
 	# before setting anything, check if the compiler is really msvc

waflib/Utils.py

@@ -409,10 +409,25 @@ def split_path_win32(path):
 		return ret
 	return re.split(re_sp, path)
 
+msysroot = None
+def split_path_msys(path):
+	if (path.startswith('/') or path.startswith('\\')) and not path.startswith('//') and not path.startswith('\\\\'):
+		# msys paths can be in the form /usr/bin
+		global msysroot
+		if not msysroot:
+			# msys has python 2.7 or 3, so we can use this
+			msysroot = subprocess.check_output(['cygpath', '-w', '/']).decode(sys.stdout.encoding or 'iso8859-1')
+			msysroot = msysroot.strip()
+		path = os.path.normpath(msysroot + os.sep + path)
+	return split_path_win32(path)
+
 if sys.platform == 'cygwin':
 	split_path = split_path_cygwin
 elif is_win32:
-	split_path = split_path_win32
+	if os.environ.get('MSYSTEM', None):
+		split_path = split_path_msys
+	else:
+		split_path = split_path_win32
 else:
 	split_path = split_path_unix
 
@@ -441,6 +456,7 @@ def check_dir(path):
 def check_exe(name, env=None):
 	"""
 	Ensure that a program exists
+
 	:type name: string
 	:param name: name or path to program
 	:return: path of the program or None
@@ -523,6 +539,25 @@ def h_fun(fun):
 			pass
 		return h
 
+def h_cmd(ins):
+	"""
+	Task command hashes are calculated by calling this function. The inputs can be
+	strings, functions, tuples/lists containing strings/functions
+	"""
+	# this function is not meant to be particularly fast
+	if isinstance(ins, str):
+		# a command is either a string
+		ret = ins
+	elif isinstance(ins, list) or isinstance(ins, tuple):
+		# or a list of functions/strings
+		ret = str([h_cmd(x) for x in ins])
+	else:
+		# or just a python function
+		ret = str(h_fun(ins))
+	if sys.hexversion > 0x3000000:
+		ret = ret.encode('iso8859-1', 'xmlcharrefreplace')
+	return ret
+
 reg_subst = re.compile(r"(\\\\)|(\$\$)|\$\{([^}]+)\}")
 def subst_vars(expr, params):
 	"""
@@ -545,6 +580,8 @@ def subst_vars(expr, params):
 			return params.get_flat(m.group(3))
 		except AttributeError:
 			return params[m.group(3)]
+		# if you get a TypeError, it means that 'expr' is not a string...
+		# Utils.subst_vars(None, env)  will not work
 	return reg_subst.sub(repl_var, expr)
 
 def destos_to_binfmt(key):
@@ -595,6 +632,9 @@ def unversioned_sys_platform():
 		return 'darwin'
 	if s == 'win32' or s == 'os2':
 		return s
+	if s == 'cli' and os.name == 'nt':
+		# ironpython is only on windows as far as we know
+		return 'win32'
 	return re.split('\d+$', s)[0]
 
 def nada(*k, **kw):
@@ -709,6 +749,7 @@ def run_once(fun):
 			cache[k] = ret
 			return ret
 	wrap.__cache__ = cache
+	wrap.__name__ = fun.__name__
 	return wrap
 
 def get_registry_app_path(key, filename):
@@ -730,3 +771,7 @@ def lib64():
 				return '64'
 	return ''
 
+def sane_path(p):
+	# private function for the time being!
+	return os.path.abspath(os.path.expanduser(p))
+

waflib/extras/build_logs.py

@@ -10,7 +10,7 @@ A system for recording all outputs to a log file. Just add the following to your
 """
 
 import atexit, sys, time, os, shutil, threading
-from waflib import Logs, Context
+from waflib import ansiterm, Logs, Context
 
 # adding the logs under the build/ directory will clash with the clean/ command
 try:
@@ -68,8 +68,12 @@ def init(ctx):
 	# sys.stdout has already been replaced, so __stdout__ will be faster
 	#sys.stdout = log_to_file(sys.stdout, fileobj, filename)
 	#sys.stderr = log_to_file(sys.stderr, fileobj, filename)
-	sys.stdout = log_to_file(sys.__stdout__, fileobj, filename)
-	sys.stderr = log_to_file(sys.__stderr__, fileobj, filename)
+	def wrap(stream):
+		if stream.isatty():
+			return ansiterm.AnsiTerm(stream)
+		return stream
+	sys.stdout = log_to_file(wrap(sys.__stdout__), fileobj, filename)
+	sys.stderr = log_to_file(wrap(sys.__stderr__), fileobj, filename)
 
 	# now mess with the logging module...
 	for x in Logs.log.handlers:

waflib/extras/c_bgxlc.py

@@ -6,7 +6,6 @@
 IBM XL Compiler for Blue Gene
 """
 
-import os
 from waflib.Tools import ccroot,ar
 from waflib.Configure import conf
 

waflib/extras/c_nec.py

@@ -6,7 +6,6 @@
 NEC SX Compiler for SX vector systems
 """
 
-import os
 import re
 from waflib import Utils
 from waflib.Tools import ccroot,ar