M7350v1_en_gpl

oe-core/scripts/contrib/bb-perf/bb-matrix-plot.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+#
+# Copyright (c) 2011, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# DESCRIPTION
+# This script operates on the .dat file generated by bb-matrix.sh. It tolerates
+# the header by skipping the first line, but error messages and bad data records
+# need to be removed first. It will generate three views of the plot, and leave
+# an interactive view open for further analysis.
+#
+# AUTHORS
+# Darren Hart <dvhart@linux.intel.com>
+#
+
+# Setup the defaults
+DATFILE="bb-matrix.dat"
+XLABEL="BB_NUMBER_THREADS"
+YLABEL="PARALLEL_MAKE"
+FIELD=3
+DEF_TITLE="Elapsed Time (seconds)"
+PM3D_FRAGMENT="unset surface; set pm3d at s hidden3d 100"
+SIZE="640,480"
+
+function usage {
+CMD=$(basename $0)
+cat <<EOM
+Usage: $CMD [-d datfile] [-f field] [-h] [-t title] [-w]
+  -d datfile    The data file generated by bb-matrix.sh (default: $DATFILE)
+  -f field      The field index to plot as the Z axis from the data file
+                (default: $FIELD, "$DEF_TITLE")
+  -h            Display this help message
+  -s W,H        PNG and window size in pixels (default: $SIZE)
+  -t title      The title to display, should describe the field (-f) and units
+                (default: "$DEF_TITLE")
+  -w            Render the plot as wireframe with a 2D colormap projected on the
+                XY plane rather than as the texture for the surface
+EOM
+}
+
+# Parse and validate arguments
+while getopts "d:f:hs:t:w" OPT; do
+	case $OPT in
+	d)
+		DATFILE="$OPTARG"
+		;;
+	f)
+		FIELD="$OPTARG"
+		;;
+	h)
+		usage
+		exit 0
+		;;
+	s)
+		SIZE="$OPTARG"
+		;;
+	t)
+		TITLE="$OPTARG"
+		;;
+	w)
+		PM3D_FRAGMENT="set pm3d at b"
+		W="-w"
+		;;
+	*)
+		usage
+		exit 1
+		;;
+	esac
+done
+
+# Ensure the data file exists
+if [ ! -f "$DATFILE" ]; then
+	echo "ERROR: $DATFILE does not exist"
+	usage
+	exit 1
+fi
+PLOT_BASENAME=${DATFILE%.*}-f$FIELD$W
+
+# Set a sane title
+# TODO: parse the header and define titles for each format parameter for TIME(1)
+if [ -z "$TITLE" ]; then
+	if [ ! "$FIELD" == "3" ]; then
+		TITLE="Field $FIELD"
+	else
+		TITLE="$DEF_TITLE"
+	fi
+fi
+
+# Determine the dgrid3d mesh dimensions size
+MIN=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 1 | sort | uniq | head -n1)
+MAX=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 1 | sort | uniq | tail -n1)
+BB_CNT=$[${MAX#*0} - $MIN + 1]
+MIN=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 2 | sort | uniq | head -n1)
+MAX=$(tail -n +2 "$DATFILE" | cut -d ' ' -f 2 | sort | uniq | tail -n1)
+PM_CNT=$[${MAX#*0} - $MIN + 1]
+
+
+(cat <<EOF
+set title "$TITLE"
+set xlabel "$XLABEL"
+set ylabel "$YLABEL"
+set style line 100 lt 5 lw 1.5
+$PM3D_FRAGMENT
+set dgrid3d $PM_CNT,$BB_CNT
+set ticslevel 0.2
+
+set term png size $SIZE
+set output "$PLOT_BASENAME.png"
+splot "$DATFILE" every ::1 using 1:2:$FIELD with lines ls 100
+
+set view 90,0
+set output "$PLOT_BASENAME-bb.png"
+replot
+
+set view 90,90
+set output "$PLOT_BASENAME-pm.png"
+replot
+
+set view 60,30
+set term wxt size $SIZE
+replot
+EOF
+) | gnuplot --persist

oe-core/scripts/contrib/bb-perf/bb-matrix.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+#
+# Copyright (c) 2011, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# DESCRIPTION
+# This script runs BB_CMD (typically building core-image-sato) for all
+# combincations of BB_RANGE and PM_RANGE values. It saves off all the console
+# logs, the buildstats directories, and creates a bb-pm-runtime.dat file which
+# can be used to postprocess the results with a plotting tool, spreadsheet, etc.
+# Before running this script, it is recommended that you pre-download all the
+# necessary sources by performing the BB_CMD once manually. It is also a good
+# idea to disable cron to avoid runtime variations caused by things like the
+# locate process. Be sure to sanitize the dat file prior to post-processing as
+# it may contain error messages or bad runs that should be removed.
+#
+# AUTHORS
+# Darren Hart <dvhart@linux.intel.com>
+#
+
+# The following ranges are appropriate for a 4 core system with 8 logical units
+BB_RANGE="04 05 06 07 08 09 10 11 12 13 14 15 16"
+PM_RANGE="04 05 06 07 08 09 10 11 12 13 14 15 16"
+
+DATADIR="bb-matrix-$$"
+BB_CMD="bitbake core-image-minimal"
+RUNTIME_LOG="$DATADIR/bb-matrix.dat"
+
+# See TIME(1) for a description of the time format parameters
+# The following all report 0: W K r s t w
+TIME_STR="%e %S %U %P %c %w %R %F %M %x"
+
+# Prepare the DATADIR
+mkdir $DATADIR
+if [ $? -ne 0 ]; then
+	echo "Failed to create $DATADIR."
+	exit 1
+fi
+
+# Add a simple header
+echo "BB PM $TIME_STR" > $RUNTIME_LOG
+for BB in $BB_RANGE; do
+	for PM in $PM_RANGE; do
+		RUNDIR="$DATADIR/$BB-$PM-build"
+		mkdir $RUNDIR
+		BB_LOG=$RUNDIR/$BB-$PM-bitbake.log
+		date
+		echo "BB=$BB PM=$PM Logging to $BB_LOG"
+
+		# Export the variables under test and run the bitbake command
+		export BB_NUMBER_THREADS=$(echo $BB | sed 's/^0*//')
+		export PARALLEL_MAKE="-j $(echo $PM | sed 's/^0*//')"
+		/usr/bin/time -f "$BB $PM $TIME_STR" -a -o $RUNTIME_LOG $BB_CMD &> $BB_LOG
+		
+		echo "  $(tail -n1 $RUNTIME_LOG)"
+		echo -n "  Cleaning up..."
+		mv tmp/buildstats $RUNDIR/$BB-$PM-buildstats
+		rm -f pseudodone &> /dev/null
+		rm -rf tmp &> /dev/null
+		rm -rf sstate-cache &> /dev/null
+		rm -rf tmp-eglibc &> /dev/null
+		echo "done"
+	done
+done

oe-core/scripts/contrib/bbvars.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# Copyright (C) Darren Hart <dvhart@linux.intel.com>, 2010
+
+
+import sys
+import getopt
+import os
+import os.path
+import re
+
+def usage():
+    print 'Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0])
+    print '  -d FILENAME         documentation file to search'
+    print '  -h, --help          display this help and exit'
+    print '  -m METADIR          meta directory to search for recipes'
+    print '  -t FILENAME         documentation config file (for doc tags)'
+    print '  -T                  Only display variables with doc tags (requires -t)'
+
+def recipe_bbvars(recipe):
+    ''' Return a unique set of every bbvar encountered in the recipe '''
+    prog = re.compile("[A-Z_]+")
+    vset = set()
+    try:
+        r = open(recipe)
+    except IOError as (errno, strerror):
+        print 'WARNING: Failed to open recipe ', recipe
+        print strerror
+
+    for line in r:
+        # Strip any comments from the line
+        line = line.rsplit('#')[0]
+        vset = vset.union(set(prog.findall(line)))
+    r.close()
+
+    bbvars = {}
+    for v in vset:
+        bbvars[v] = 1
+
+    return bbvars
+
+def collect_bbvars(metadir):
+    ''' Walk the metadir and collect the bbvars from each recipe found '''
+    bbvars = {}
+    for root,dirs,files in os.walk(metadir):
+        for name in files:
+            if name.find(".bb") >= 0:
+                for key in recipe_bbvars(os.path.join(root,name)).iterkeys():
+                    if bbvars.has_key(key):
+                        bbvars[key] = bbvars[key] + 1
+                    else:
+                        bbvars[key] = 1
+    return bbvars
+
+def bbvar_is_documented(var, docfiles):
+    prog = re.compile(".*($|[^A-Z_])%s([^A-Z_]|$)" % (var))
+    for doc in docfiles:
+        try:
+            f = open(doc)
+        except IOError as (errno, strerror):
+            print 'WARNING: Failed to open doc ', doc
+            print strerror
+        for line in f:
+            if prog.match(line):
+                return True
+        f.close()
+    return False
+
+def bbvar_doctag(var, docconf):
+    prog = re.compile('^%s\[doc\] *= *"(.*)"' % (var))
+    if docconf == "":
+        return "?"
+
+    try:
+        f = open(docconf)
+    except IOError as (errno, strerror):
+        return strerror
+
+    for line in f:
+        m = prog.search(line)
+        if m:
+            return m.group(1)
+
+    f.close()
+    return ""
+
+def main():
+    docfiles = []
+    metadirs = []
+    bbvars = {}
+    undocumented = []
+    docconf = ""
+    onlydoctags = False
+
+    # Collect and validate input
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"])
+    except getopt.GetoptError, err:
+        print '%s' % str(err)
+        usage()
+        sys.exit(2)
+
+    for o, a in opts:
+        if o in ('-h', '--help'):
+            usage()
+            sys.exit(0)
+        elif o == '-d':
+            if os.path.isfile(a):
+                docfiles.append(a)
+            else:
+                print 'ERROR: documentation file %s is not a regular file' % (a)
+                sys.exit(3)
+        elif o == '-m':
+            if os.path.isdir(a):
+                metadirs.append(a)
+            else:
+                print 'ERROR: meta directory %s is not a directory' % (a)
+                sys.exit(4)
+        elif o == "-t":
+            if os.path.isfile(a):
+                docconf = a
+        elif o == "-T":
+            onlydoctags = True
+        else:
+            assert False, "unhandled option"
+
+    if len(docfiles) == 0:
+        print 'ERROR: no docfile specified'
+        usage()
+        sys.exit(5)
+
+    if len(metadirs) == 0:
+        print 'ERROR: no metadir specified'
+        usage()
+        sys.exit(6)
+
+    if onlydoctags and docconf == "":
+        print 'ERROR: no docconf specified'
+        usage()
+        sys.exit(7)
+
+    # Collect all the variable names from the recipes in the metadirs
+    for m in metadirs:
+        for key,cnt in collect_bbvars(m).iteritems():
+            if bbvars.has_key(key):
+                bbvars[key] = bbvars[key] + cnt
+            else:
+                bbvars[key] = cnt
+
+    # Check each var for documentation
+    varlen = 0
+    for v in bbvars.iterkeys():
+        if len(v) > varlen:
+            varlen = len(v)
+        if not bbvar_is_documented(v, docfiles):
+            undocumented.append(v)
+    undocumented.sort()
+    varlen = varlen + 1
+
+    # Report all undocumented variables
+    print 'Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars))
+    header = '%s%s%s' % (str("VARIABLE").ljust(varlen), str("COUNT").ljust(6), str("DOCTAG").ljust(7))
+    print header
+    print str("").ljust(len(header), '=')
+    for v in undocumented:
+        doctag = bbvar_doctag(v, docconf)
+        if not onlydoctags or not doctag == "":
+            print '%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag)
+
+
+if __name__ == "__main__":
+    main()

oe-core/scripts/contrib/documentation-audit.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# Perform an audit of which packages provide documentation and which
+# are missing -doc packages.
+#
+# Setup requirements: be sure to be building for MACHINE=qemux86. Run
+# this script after source'ing the build environment script, so you're
+# running it from build/ directory.
+#
+# Maintainer: Scott Garman <scott.a.garman@intel.com>
+
+REPORT_DOC_SIMPLE="documentation_exists.txt"
+REPORT_DOC_DETAIL="documentation_exists_detail.txt"
+REPORT_MISSING_SIMPLE="documentation_missing.txt"
+REPORT_MISSING_DETAIL="documentation_missing_detail.txt"
+REPORT_BUILD_ERRORS="build_errors.txt"
+
+rm -rf $REPORT_DOC_SIMPLE $REPORT_DOC_DETAIL $REPORT_MISSING_SIMPLE $REPORT_MISSING_DETAIL
+
+BITBAKE=`which bitbake`
+if [ -z "$BITBAKE" ]; then
+	echo "Error: bitbake command not found."
+	echo "Did you forget to source the build environment script?"
+	exit 1
+fi
+
+echo "REMINDER: you need to build for MACHINE=qemux86 or you won't get useful results"
+echo "REMINDER: you need to set LICENSE_FLAGS_WHITELIST appropriately in local.conf or "
+echo " you'll get false positives.  For example, LICENSE_FLAGS_WHITELIST = \"Commercial\""
+
+for pkg in `bitbake -s | awk '{ print \$1 }'`; do
+	if [[ "$pkg" == "Loading" || "$pkg" == "Loaded" ||
+          "$pkg" == "Parsing" || "$pkg" == "Package" ||
+          "$pkg" == "NOTE:"   || "$pkg" == "WARNING:" ||
+          "$pkg" == "done."   || "$pkg" == "============" ]]
+	then
+		# Skip initial bitbake output
+		continue
+	fi
+	if [[ "$pkg" =~ -native$ || "$pkg" =~ -nativesdk$ ||
+          "$pkg" =~ -cross-canadian ]]; then
+		# Skip native/nativesdk/cross-canadian recipes
+		continue
+	fi
+	if [[ "$pkg" =~ ^meta- || "$pkg" =~ ^task- || "$pkg" =~ -image ]]; then
+		# Skip meta, task and image recipes
+		continue
+	fi
+	if [[ "$pkg" =~ ^glibc- || "$pkg" =~ ^libiconv$ ||
+          "$pkg" =~ -toolchain$ || "$pkg" =~ ^package-index$ ||
+          "$pkg" =~ ^linux- || "$pkg" =~ ^adt-installer$ ||
+          "$pkg" =~ ^eds-tools$ || "$pkg" =~ ^external-python-tarball$ ||
+          "$pkg" =~ ^qt4-embedded$ || "$pkg" =~ ^qt-mobility ]]; then
+		# Skip glibc, libiconv, -toolchain, and other recipes known
+		# to cause build conflicts or trigger false positives.
+		continue
+	fi	
+
+	echo "Building package $pkg..."
+	bitbake $pkg > /dev/null
+	if [ $? -ne 0 ]; then
+		echo "There was an error building package $pkg" >> "$REPORT_MISSING_DETAIL"
+		echo "$pkg" >> $REPORT_BUILD_ERRORS
+
+		# Do not skip the remaining tests, as sometimes the
+		# exit status is 1 due to QA errors, and we can still
+		# perform the -doc checks.
+	fi
+
+	echo "$pkg built successfully, checking for a documentation package..."
+	WORKDIR=`bitbake -e $pkg | grep ^WORKDIR | awk -F '=' '{ print \$2 }' | awk -F '"' '{ print \$2 }'`
+	FIND_DOC_PKG=`find $WORKDIR/packages-split/*-doc -maxdepth 0 -type d`
+	if [ -z "$FIND_DOC_PKG" ]; then
+		# No -doc package was generated:
+		echo "No -doc package: $pkg" >> "$REPORT_MISSING_DETAIL"
+		echo "$pkg" >> $REPORT_MISSING_SIMPLE
+		continue
+	fi
+
+	FIND_DOC_FILES=`find $FIND_DOC_PKG -type f`
+	if [ -z "$FIND_DOC_FILES" ]; then
+		# No files shipped with the -doc package:
+		echo "No files shipped with the -doc package: $pkg" >> "$REPORT_MISSING_DETAIL"
+		echo "$pkg" >> $REPORT_MISSING_SIMPLE
+		continue
+	fi
+
+	echo "Documentation shipped with $pkg:" >> "$REPORT_DOC_DETAIL"
+	echo "$FIND_DOC_FILES" >> "$REPORT_DOC_DETAIL"
+	echo ""	>> "$REPORT_DOC_DETAIL"
+
+	echo "$pkg" >> "$REPORT_DOC_SIMPLE"
+done

oe-core/scripts/contrib/python/generate-manifest-2.7.py

@@ -0,0 +1,379 @@
+#!/usr/bin/env python
+
+# generate Python Manifest for the OpenEmbedded build system
+# (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de>
+# (C) 2007 Jeremy Laine
+# licensed under MIT, see COPYING.MIT
+#
+# June 22, 2011 -- Mark Hatle <mark.hatle@windriver.com>
+#  * Updated to no longer generate special -dbg package, instead use the
+#    single system -dbg
+#  * Update version with ".1" to indicate this change
+
+import os
+import sys
+import time
+
+VERSION = "2.7.2"
+
+__author__ = "Michael 'Mickey' Lauer <mlauer@vanille-media.de>"
+__version__ = "20110222.2"
+
+class MakefileMaker:
+
+    def __init__( self, outfile ):
+        """initialize"""
+        self.packages = {}
+        self.targetPrefix = "${libdir}/python%s/" % VERSION[:3]
+        self.output = outfile
+        self.out( """
+# WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file.
+# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de>
+# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy
+""" % ( sys.argv[0], __version__ ) )
+
+    #
+    # helper functions
+    #
+
+    def out( self, data ):
+        """print a line to the output file"""
+        self.output.write( "%s\n" % data )
+
+    def setPrefix( self, targetPrefix ):
+        """set a file prefix for addPackage files"""
+        self.targetPrefix = targetPrefix
+
+    def doProlog( self ):
+        self.out( """ """ )
+        self.out( "" )
+
+    def addPackage( self, name, description, dependencies, filenames ):
+        """add a package to the Makefile"""
+        if type( filenames ) == type( "" ):
+            filenames = filenames.split()
+        fullFilenames = []
+        for filename in filenames:
+            if filename[0] != "$":
+                fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) )
+            else:
+                fullFilenames.append( filename )
+        self.packages[name] = description, dependencies, fullFilenames
+
+    def doBody( self ):
+        """generate body of Makefile"""
+
+        global VERSION
+
+        #
+        # generate provides line
+        #
+
+        provideLine = 'PROVIDES+="'
+        for name in sorted(self.packages):
+            provideLine += "%s " % name
+        provideLine += '"'
+
+        self.out( provideLine )
+        self.out( "" )
+
+        #
+        # generate package line
+        #
+
+        packageLine = 'PACKAGES="${PN}-dbg '
+        for name in sorted(self.packages):
+            if name != '${PN}-dbg':
+                packageLine += "%s " % name
+        packageLine += '${PN}-modules"'
+
+        self.out( packageLine )
+        self.out( "" )
+
+        #
+        # generate package variables
+        #
+
+        for name, data in sorted(self.packages.iteritems()):
+            desc, deps, files = data
+
+            #
+            # write out the description, revision and dependencies
+            #
+            self.out( 'DESCRIPTION_%s="%s"' % ( name, desc ) )
+            self.out( 'RDEPENDS_%s="%s"' % ( name, deps ) )
+
+            line = 'FILES_%s="' % name
+
+            #
+            # check which directories to make in the temporary directory
+            #
+
+            dirset = {} # if python had a set-datatype this would be sufficient. for now, we're using a dict instead.
+            for target in files:
+                dirset[os.path.dirname( target )] = True
+
+            #
+            # generate which files to copy for the target (-dfR because whole directories are also allowed)
+            #
+
+            for target in files:
+                line += "%s " % target
+
+            line += '"'
+            self.out( line )
+            self.out( "" )
+
+        self.out( 'DESCRIPTION_${PN}-modules="All Python modules"' )
+        line = 'RDEPENDS_${PN}-modules="'
+
+        for name, data in sorted(self.packages.iteritems()):
+            if name not in ['${PN}-dev']:
+                line += "%s " % name
+
+        self.out( "%s \"" % line )
+        self.out( 'ALLOW_EMPTY_${PN}-modules = "1"' )
+
+    def doEpilog( self ):
+        self.out( """""" )
+        self.out( "" )
+
+    def make( self ):
+        self.doProlog()
+        self.doBody()
+        self.doEpilog()
+
+if __name__ == "__main__":
+
+    if len( sys.argv ) > 1:
+        os.popen( "rm -f ./%s" % sys.argv[1] )
+        outfile = file( sys.argv[1], "w" )
+    else:
+        outfile = sys.stdout
+
+    m = MakefileMaker( outfile )
+
+    # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies!
+    # Parameters: revision, name, description, dependencies, filenames
+    #
+
+    m.addPackage( "${PN}-core", "Python Interpreter and core modules (needed!)", "",
+    "__future__.* _abcoll.* abc.* copy.* copy_reg.* ConfigParser.* " +
+    "genericpath.* getopt.* linecache.* new.* " +
+    "os.* posixpath.* struct.* " +
+    "warnings.* site.* stat.* " +
+    "UserDict.* UserList.* UserString.* " +
+    "lib-dynload/binascii.so lib-dynload/_struct.so lib-dynload/time.so " +
+    "lib-dynload/xreadlines.so types.* platform.* ${bindir}/python* "  + 
+    "_weakrefset.* sysconfig.* config/Makefile " +
+    "${includedir}/python${PYTHON_MAJMIN}/pyconfig.h " +
+    "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py ")
+
+    m.addPackage( "${PN}-dev", "Python Development Package", "${PN}-core",
+    "${includedir} " +
+    "${libdir}/lib*${SOLIBSDEV} " +
+    "${libdir}/*.la " +
+    "${libdir}/*.a " +
+    "${libdir}/*.o " +
+    "${libdir}/pkgconfig " +
+    "${base_libdir}/*.a " +
+    "${base_libdir}/*.o " +
+    "${datadir}/aclocal " +
+    "${datadir}/pkgconfig " )
+
+    m.addPackage( "${PN}-2to3", "Python Automated Python 2 to 3 code translation", "${PN}-core",
+    "${bindir}/2to3 lib2to3" ) # package
+
+    m.addPackage( "${PN}-idle", "Python Integrated Development Environment", "${PN}-core ${PN}-tkinter",
+    "${bindir}/idle idlelib" ) # package
+
+    m.addPackage( "${PN}-pydoc", "Python Interactive Help Support", "${PN}-core ${PN}-lang ${PN}-stringold ${PN}-re",
+    "${bindir}/pydoc pydoc.* pydoc_data" )
+
+    m.addPackage( "${PN}-smtpd", "Python Simple Mail Transport Daemon", "${PN}-core ${PN}-netserver ${PN}-email ${PN}-mime",
+    "${bindir}/smtpd.* smtpd.*" )
+
+    m.addPackage( "${PN}-audio", "Python Audio Handling", "${PN}-core",
+    "wave.* chunk.* sndhdr.* lib-dynload/ossaudiodev.so lib-dynload/audioop.so audiodev.* sunaudio.* sunau.* toaiff.*" )
+
+    m.addPackage( "${PN}-bsddb", "Python Berkeley Database Bindings", "${PN}-core",
+    "bsddb lib-dynload/_bsddb.so" ) # package
+
+    m.addPackage( "${PN}-codecs", "Python Codecs, Encodings & i18n Support", "${PN}-core ${PN}-lang",
+    "codecs.* encodings gettext.* locale.* lib-dynload/_locale.so lib-dynload/_codecs* lib-dynload/_multibytecodec.so lib-dynload/unicodedata.so stringprep.* xdrlib.*" )
+
+    m.addPackage( "${PN}-compile", "Python Bytecode Compilation Support", "${PN}-core",
+    "py_compile.* compileall.*" )
+
+    m.addPackage( "${PN}-compiler", "Python Compiler Support", "${PN}-core",
+    "compiler" ) # package
+
+    m.addPackage( "${PN}-compression", "Python High Level Compression Support", "${PN}-core ${PN}-zlib",
+    "gzip.* zipfile.* tarfile.* lib-dynload/bz2.so" )
+
+    m.addPackage( "${PN}-crypt", "Python Basic Cryptographic and Hashing Support", "${PN}-core",
+    "hashlib.* md5.* sha.* lib-dynload/crypt.so lib-dynload/_hashlib.so lib-dynload/_sha256.so lib-dynload/_sha512.so" )
+
+    m.addPackage( "${PN}-textutils", "Python Option Parsing, Text Wrapping and Comma-Separated-Value Support", "${PN}-core ${PN}-io ${PN}-re ${PN}-stringold",
+    "lib-dynload/_csv.so csv.* optparse.* textwrap.*" )
+
+    m.addPackage( "${PN}-curses", "Python Curses Support", "${PN}-core",
+    "curses lib-dynload/_curses.so lib-dynload/_curses_panel.so" ) # directory + low level module
+
+    m.addPackage( "${PN}-ctypes", "Python C Types Support", "${PN}-core",
+    "ctypes lib-dynload/_ctypes.so lib-dynload/_ctypes_test.so" ) # directory + low level module
+
+    m.addPackage( "${PN}-datetime", "Python Calendar and Time support", "${PN}-core ${PN}-codecs",
+    "_strptime.* calendar.* lib-dynload/datetime.so" )
+
+    m.addPackage( "${PN}-db", "Python File-Based Database Support", "${PN}-core",
+    "anydbm.* dumbdbm.* whichdb.* " )
+
+    m.addPackage( "${PN}-debugger", "Python Debugger", "${PN}-core ${PN}-io ${PN}-lang ${PN}-re ${PN}-stringold ${PN}-shell ${PN}-pprint",
+    "bdb.* pdb.*" )
+
+    m.addPackage( "${PN}-difflib", "Python helpers for computing deltas between objects.", "${PN}-lang ${PN}-re",
+    "difflib.*" )
+
+    m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core",
+    "config distutils" ) # package
+
+    m.addPackage( "${PN}-doctest", "Python framework for running examples in docstrings.", "${PN}-core ${PN}-lang ${PN}-io ${PN}-re ${PN}-unittest ${PN}-debugger ${PN}-difflib",
+    "doctest.*" )
+
+    # FIXME consider adding to some higher level package
+    m.addPackage( "${PN}-elementtree", "Python elementree", "${PN}-core",
+    "lib-dynload/_elementtree.so" )
+
+    m.addPackage( "${PN}-email", "Python Email Support", "${PN}-core ${PN}-io ${PN}-re ${PN}-mime ${PN}-audio ${PN}-image ${PN}-netclient",
+    "imaplib.* email" ) # package
+
+    m.addPackage( "${PN}-fcntl", "Python's fcntl Interface", "${PN}-core",
+    "lib-dynload/fcntl.so" )
+
+    m.addPackage( "${PN}-hotshot", "Python Hotshot Profiler", "${PN}-core",
+    "hotshot lib-dynload/_hotshot.so" )
+
+    m.addPackage( "${PN}-html", "Python HTML Processing", "${PN}-core",
+    "formatter.* htmlentitydefs.* htmllib.* markupbase.* sgmllib.* HTMLParser.* " )
+
+    m.addPackage( "${PN}-gdbm", "Python GNU Database Support", "${PN}-core",
+    "lib-dynload/gdbm.so" )
+
+    m.addPackage( "${PN}-image", "Python Graphical Image Handling", "${PN}-core",
+    "colorsys.* imghdr.* lib-dynload/imageop.so lib-dynload/rgbimg.so" )
+
+    m.addPackage( "${PN}-io", "Python Low-Level I/O", "${PN}-core ${PN}-math",
+    "lib-dynload/_socket.so lib-dynload/_io.so lib-dynload/_ssl.so lib-dynload/select.so lib-dynload/termios.so lib-dynload/cStringIO.so " +
+    "pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" )
+
+    m.addPackage( "${PN}-json", "Python JSON Support", "${PN}-core ${PN}-math ${PN}-re",
+    "json lib-dynload/_json.so" ) # package
+
+    m.addPackage( "${PN}-lang", "Python Low-Level Language Support", "${PN}-core",
+    "lib-dynload/_bisect.so lib-dynload/_collections.so lib-dynload/_heapq.so lib-dynload/_weakref.so lib-dynload/_functools.so " +
+    "lib-dynload/array.so lib-dynload/itertools.so lib-dynload/operator.so lib-dynload/parser.so " +
+    "atexit.* bisect.* code.* codeop.* collections.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* symbol.* repr.* token.* " +
+    "tokenize.* traceback.* weakref.*" )
+
+    m.addPackage( "${PN}-logging", "Python Logging Support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-stringold",
+    "logging" ) # package
+
+    m.addPackage( "${PN}-mailbox", "Python Mailbox Format Support", "${PN}-core ${PN}-mime",
+    "mailbox.*" )
+
+    m.addPackage( "${PN}-math", "Python Math Support", "${PN}-core",
+    "lib-dynload/cmath.so lib-dynload/math.so lib-dynload/_random.so random.* sets.*" )
+
+    m.addPackage( "${PN}-mime", "Python MIME Handling APIs", "${PN}-core ${PN}-io",
+    "mimetools.* uu.* quopri.* rfc822.* MimeWriter.*" )
+
+    m.addPackage( "${PN}-mmap", "Python Memory-Mapped-File Support", "${PN}-core ${PN}-io",
+    "lib-dynload/mmap.so " )
+
+    m.addPackage( "${PN}-multiprocessing", "Python Multiprocessing Support", "${PN}-core ${PN}-io ${PN}-lang",
+    "lib-dynload/_multiprocessing.so multiprocessing" ) # package
+
+    m.addPackage( "${PN}-netclient", "Python Internet Protocol Clients", "${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime",
+    "*Cookie*.* " +
+    "base64.* cookielib.* ftplib.* gopherlib.* hmac.* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib.* urllib2.* urlparse.* uuid.* rfc822.* mimetools.*" )
+
+    m.addPackage( "${PN}-netserver", "Python Internet Protocol Servers", "${PN}-core ${PN}-netclient",
+    "cgi.* *HTTPServer.* SocketServer.*" )
+
+    m.addPackage( "${PN}-numbers", "Python Number APIs", "${PN}-core ${PN}-lang ${PN}-re",
+    "decimal.* numbers.*" )
+
+    m.addPackage( "${PN}-pickle", "Python Persistence Support", "${PN}-core ${PN}-codecs ${PN}-io ${PN}-re",
+    "pickle.* shelve.* lib-dynload/cPickle.so pickletools.*" )
+
+    m.addPackage( "${PN}-pkgutil", "Python Package Extension Utility Support", "${PN}-core",
+    "pkgutil.*")
+
+    m.addPackage( "${PN}-pprint", "Python Pretty-Print Support", "${PN}-core",
+    "pprint.*" )
+
+    m.addPackage( "${PN}-profile", "Python Basic Profiling Support", "${PN}-core ${PN}-textutils",
+    "profile.* pstats.* cProfile.* lib-dynload/_lsprof.so" )
+
+    m.addPackage( "${PN}-re", "Python Regular Expression APIs", "${PN}-core",
+    "re.* sre.* sre_compile.* sre_constants* sre_parse.*" ) # _sre is builtin
+
+    m.addPackage( "${PN}-readline", "Python Readline Support", "${PN}-core",
+    "lib-dynload/readline.so rlcompleter.*" )
+
+    m.addPackage( "${PN}-resource", "Python Resource Control Interface", "${PN}-core",
+    "lib-dynload/resource.so" )
+
+    m.addPackage( "${PN}-shell", "Python Shell-Like Functionality", "${PN}-core ${PN}-re",
+    "cmd.* commands.* dircache.* fnmatch.* glob.* popen2.* shlex.* shutil.*" )
+
+    m.addPackage( "${PN}-robotparser", "Python robots.txt parser", "${PN}-core ${PN}-netclient",
+    "robotparser.*")
+
+    m.addPackage( "${PN}-subprocess", "Python Subprocess Support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle",
+    "subprocess.*" )
+
+    m.addPackage( "${PN}-sqlite3", "Python Sqlite3 Database Support", "${PN}-core ${PN}-datetime ${PN}-lang ${PN}-crypt ${PN}-io ${PN}-threading ${PN}-zlib",
+    "lib-dynload/_sqlite3.so sqlite3/dbapi2.* sqlite3/__init__.* sqlite3/dump.*" )
+
+    m.addPackage( "${PN}-sqlite3-tests", "Python Sqlite3 Database Support Tests", "${PN}-core ${PN}-sqlite3",
+    "sqlite3/test" )
+
+    m.addPackage( "${PN}-stringold", "Python String APIs [deprecated]", "${PN}-core ${PN}-re",
+    "lib-dynload/strop.so string.* stringold.*" )
+
+    m.addPackage( "${PN}-syslog", "Python Syslog Interface", "${PN}-core",
+    "lib-dynload/syslog.so" )
+
+    m.addPackage( "${PN}-terminal", "Python Terminal Controlling Support", "${PN}-core ${PN}-io",
+    "pty.* tty.*" )
+
+    m.addPackage( "${PN}-tests", "Python Tests", "${PN}-core",
+    "test" ) # package
+
+    m.addPackage( "${PN}-threading", "Python Threading & Synchronization Support", "${PN}-core ${PN}-lang",
+    "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* Queue.*" )
+
+    m.addPackage( "${PN}-tkinter", "Python Tcl/Tk Bindings", "${PN}-core",
+    "lib-dynload/_tkinter.so lib-tk" ) # package
+
+    m.addPackage( "${PN}-unittest", "Python Unit Testing Framework", "${PN}-core ${PN}-stringold ${PN}-lang",
+    "unittest/" )
+
+    m.addPackage( "${PN}-unixadmin", "Python Unix Administration Support", "${PN}-core",
+    "lib-dynload/nis.so lib-dynload/grp.so lib-dynload/pwd.so getpass.*" )
+
+    m.addPackage( "${PN}-xml", "Python basic XML support.", "${PN}-core ${PN}-elementtree ${PN}-re",
+    "lib-dynload/pyexpat.so xml xmllib.*" ) # package
+
+    m.addPackage( "${PN}-xmlrpc", "Python XMLRPC Support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang",
+    "xmlrpclib.* SimpleXMLRPCServer.* DocXMLRPCServer.*" )
+
+    m.addPackage( "${PN}-zlib", "Python zlib Support.", "${PN}-core",
+    "lib-dynload/zlib.so" )
+
+    m.addPackage( "${PN}-mailbox", "Python Mailbox Format Support", "${PN}-core ${PN}-mime",
+    "mailbox.*" )
+
+    m.make()

oe-core/scripts/contrib/test_build_time.sh

@@ -0,0 +1,237 @@
+#!/bin/bash
+
+# Build performance regression test script
+#
+# Copyright 2011 Intel Corporation
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+#
+# DESCRIPTION
+# This script is intended to be used in conjunction with "git bisect run"
+# in order to find regressions in build time, however it can also be used
+# independently. It cleans out the build output directories, runs a
+# specified worker script (an example is test_build_time_worker.sh) under
+# TIME(1), logs the results to TEST_LOGDIR (default /tmp) and returns a
+# value telling "git bisect run" whether the build time is good (under
+# the specified threshold) or bad (over it). There is also a tolerance
+# option but it is not particularly useful as it only subtracts the
+# tolerance from the given threshold and uses it as the actual threshold.
+#
+# It is also capable of taking a file listing git revision hashes to be
+# test-applied to the repository in order to get past build failures that
+# would otherwise cause certain revisions to have to be skipped; if a
+# revision does not apply cleanly then the script assumes it does not
+# need to be applied and ignores it.
+#
+# Please see the help output (syntax below) for some important setup
+# instructions.
+#
+# AUTHORS
+# Paul Eggleton <paul.eggleton@linux.intel.com>
+
+
+syntax() {
+    echo "syntax: $0 <script> <time> <tolerance> [patchrevlist]"
+    echo ""
+    echo "  script       - worker script file (if in current dir, prefix with ./)"
+    echo "  time         - time threshold (in seconds, suffix m for minutes)"
+    echo "  tolerance    - tolerance (in seconds, suffix m for minutes or % for"
+    echo "                 percentage, can be 0)"
+    echo "  patchrevlist - optional file listing revisions to apply as patches on top"
+    echo ""
+    echo "You must set TEST_BUILDDIR to point to a previously created build directory,"
+    echo "however please note that this script will wipe out the TMPDIR defined in"
+    echo "TEST_BUILDDIR/conf/local.conf as part of its initial setup (as well as your"
+    echo "~/.ccache)"
+    echo ""
+    echo "To get rid of the sudo prompt, please add the following line to /etc/sudoers"
+    echo "(use 'visudo' to edit this; also it is assumed that the user you are running"
+    echo "as is a member of the 'wheel' group):"
+    echo ""
+    echo "%wheel ALL=(ALL) NOPASSWD: /sbin/sysctl -w vm.drop_caches=[1-3]"
+    echo ""
+    echo "Note: it is recommended that you disable crond and any other process that"
+    echo "may cause significant CPU or I/O usage during build performance tests."
+}
+
+# Note - we exit with 250 here because that will tell git bisect run that
+# something bad happened and stop
+if [ "$1" = "" ] ; then
+   syntax
+   exit 250
+fi
+
+if [ "$2" = "" ] ; then
+   syntax
+   exit 250
+fi
+
+if [ "$3" = "" ] ; then
+   syntax
+   exit 250
+fi
+
+if ! [[ "$2" =~ ^[0-9][0-9m.]*$ ]] ; then
+   echo "'$2' is not a valid number for threshold"
+   exit 250
+fi
+
+if ! [[ "$3" =~ ^[0-9][0-9m.%]*$ ]] ; then
+   echo "'$3' is not a valid number for tolerance"
+   exit 250
+fi
+
+if [ "$TEST_BUILDDIR" = "" ] ; then
+   echo "Please set TEST_BUILDDIR to a previously created build directory"
+   exit 250
+fi
+
+if [ ! -d "$TEST_BUILDDIR" ] ; then
+   echo "TEST_BUILDDIR $TEST_BUILDDIR not found"
+   exit 250
+fi
+
+git diff --quiet
+if [ $? != 0 ] ; then
+    echo "Working tree is dirty, cannot proceed"
+    exit 251
+fi
+
+if [ "$BB_ENV_EXTRAWHITE" != "" ] ; then
+   echo "WARNING: you are running after sourcing the build environment script, this is not recommended"
+fi
+
+runscript=$1
+timethreshold=$2
+tolerance=$3
+
+if [ "$4" != "" ] ; then
+    patchrevlist=`cat $4`
+else
+    patchrevlist=""
+fi
+
+if [[ timethreshold == *m* ]] ; then
+    timethreshold=`echo $timethreshold | sed s/m/*60/ | bc`
+fi
+
+if [[ $tolerance == *m* ]] ; then
+    tolerance=`echo $tolerance | sed s/m/*60/ | bc`
+elif [[ $tolerance == *%* ]] ; then
+    tolerance=`echo $tolerance | sed s/%//`
+    tolerance=`echo "scale = 2; (($tolerance * $timethreshold) / 100)" | bc`
+fi
+
+tmpdir=`grep "^TMPDIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/TMPDIR[ \t]*=[ \t\?]*"//' -e 's/"//'`
+if [ "x$tmpdir" = "x" ]; then
+    echo "Unable to determine TMPDIR from $TEST_BUILDDIR/conf/local.conf, bailing out"
+    exit 250
+fi
+sstatedir=`grep "^SSTATE_DIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/SSTATE_DIR[ \t\?]*=[ \t]*"//' -e 's/"//'`
+if [ "x$sstatedir" = "x" ]; then
+    echo "Unable to determine SSTATE_DIR from $TEST_BUILDDIR/conf/local.conf, bailing out"
+    exit 250
+fi
+
+if [ `expr length $tmpdir` -lt 4 ] ; then
+    echo "TMPDIR $tmpdir is less than 4 characters, bailing out"
+    exit 250
+fi
+
+if [ `expr length $sstatedir` -lt 4 ] ; then
+    echo "SSTATE_DIR $sstatedir is less than 4 characters, bailing out"
+    exit 250
+fi
+
+echo -n "About to wipe out TMPDIR $tmpdir, press Ctrl+C to break out...  "
+for i in 9 8 7 6 5 4 3 2 1
+do
+    echo -ne "\x08$i"
+    sleep 1
+done
+echo
+
+pushd . > /dev/null
+
+rm -f pseudodone
+echo "Removing TMPDIR $tmpdir..."
+rm -rf $tmpdir
+echo "Removing TMPDIR $tmpdir-*libc..."
+rm -rf $tmpdir-*libc
+echo "Removing SSTATE_DIR $sstatedir..."
+rm -rf $sstatedir
+echo "Removing ~/.ccache..."
+rm -rf ~/.ccache
+
+echo "Syncing..."
+sync
+sync
+echo "Dropping VM cache..."
+#echo 3 > /proc/sys/vm/drop_caches
+sudo /sbin/sysctl -w vm.drop_caches=3 > /dev/null
+
+if [ "$TEST_LOGDIR" = "" ] ; then
+    logdir="/tmp"
+else
+    logdir="$TEST_LOGDIR"
+fi
+rev=`git rev-parse HEAD`
+logfile="$logdir/timelog_$rev.log"
+echo -n > $logfile
+
+gitroot=`git rev-parse --show-toplevel`
+cd $gitroot
+for patchrev in $patchrevlist ; do
+    echo "Applying $patchrev"
+    patchfile=`mktemp`
+    git show $patchrev > $patchfile
+    git apply --check $patchfile &> /dev/null
+    if [ $? != 0 ] ; then
+        echo " ... patch does not apply without errors, ignoring"
+    else
+        echo "Applied $patchrev" >> $logfile
+        git apply $patchfile &> /dev/null
+    fi
+    rm $patchfile
+done
+
+sync
+echo "Quiescing for 5s..."
+sleep 5
+
+echo "Running $runscript at $rev..."
+timeoutfile=`mktemp`
+/usr/bin/time -o $timeoutfile -f "%e\nreal\t%E\nuser\t%Us\nsys\t%Ss\nmaxm\t%Mk" $runscript 2>&1 | tee -a $logfile
+exitstatus=$PIPESTATUS
+
+git reset --hard HEAD > /dev/null
+popd > /dev/null
+
+timeresult=`head -n1 $timeoutfile`
+cat $timeoutfile | tee -a $logfile
+rm $timeoutfile
+
+if [ $exitstatus != 0 ] ; then
+    # Build failed, exit with 125 to tell git bisect run to skip this rev
+    echo "*** Build failed (exit code $exitstatus), skipping..." | tee -a $logfile
+    exit 125
+fi
+
+ret=`echo "scale = 2; $timeresult > $timethreshold - $tolerance" | bc`
+echo "Returning $ret" | tee -a $logfile
+exit $ret
+

oe-core/scripts/contrib/test_build_time_worker.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# This is an example script to be used in conjunction with test_build_time.sh
+
+if [ "$TEST_BUILDDIR" = "" ] ; then
+    echo "TEST_BUILDDIR is not set"
+    exit 1
+fi
+
+buildsubdir=`basename $TEST_BUILDDIR`
+if [ ! -d $buildsubdir ] ; then
+    echo "Unable to find build subdir $buildsubdir in current directory"
+    exit 1
+fi
+
+if [ -f oe-init-build-env ] ; then
+    . ./oe-init-build-env $buildsubdir
+elif [ -f poky-init-build-env ] ; then
+    . ./poky-init-build-env $buildsubdir
+else
+    echo "Unable to find build environment setup script"
+    exit 1
+fi
+
+if [ -f ../meta/recipes-sato/images/core-image-sato.bb ] ; then
+    target="core-image-sato"
+else
+    target="poky-image-sato"
+fi
+
+echo "Build started at `date "+%Y-%m-%d %H:%M:%S"`"
+echo "bitbake $target"
+bitbake $target
+ret=$?
+echo "Build finished at `date "+%Y-%m-%d %H:%M:%S"`"
+exit $ret
+