M7350v1_en_gpl

oe-core/meta/lib/oe/buildhistory_analysis.py

@@ -0,0 +1,295 @@
+# Report significant differences in the buildhistory repository since a specific revision
+#
+# Copyright (C) 2012 Intel Corporation
+# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Note: requires GitPython 0.3.1+
+#
+# You can use this from the command line by running scripts/buildhistory-diff
+#
+
+import sys
+import os.path
+import difflib
+import git
+
+
+# How to display fields
+list_fields = ['DEPENDS', 'RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS']
+numeric_fields = ['PKGSIZE', 'IMAGESIZE']
+# Fields to monitor
+monitor_fields = ['RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE']
+# Percentage change to alert for numeric fields
+monitor_numeric_threshold = 20
+# Image files to monitor (note that image-info.txt is handled separately)
+img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
+# Related context fields for reporting (note: PE, PV & PR are always reported for monitored package fields)
+related_fields = {}
+related_fields['RDEPENDS'] = ['DEPENDS']
+related_fields['RRECOMMENDS'] = ['DEPENDS']
+related_fields['FILELIST'] = ['FILES']
+related_fields['PKGSIZE'] = ['FILELIST']
+related_fields['files-in-image.txt'] = ['installed-package-names.txt', 'USER_CLASSES', 'IMAGE_CLASSES', 'ROOTFS_POSTPROCESS_COMMAND', 'IMAGE_POSTPROCESS_COMMAND']
+related_fields['installed-package-names.txt'] = ['IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS']
+
+
+class ChangeRecord:
+    def __init__(self, path, fieldname, oldvalue, newvalue, monitored):
+        self.path = path
+        self.fieldname = fieldname
+        self.oldvalue = oldvalue
+        self.newvalue = newvalue
+        self.monitored = monitored
+        self.related = []
+        self.filechanges = None
+
+    def __str__(self):
+        return self._str_internal(True)
+
+    def _str_internal(self, pathprefix):
+        if pathprefix:
+            prefix = '%s: ' % self.path
+        else:
+            prefix = ''
+
+        if self.fieldname in list_fields:
+            aitems = self.oldvalue.split()
+            bitems = self.newvalue.split()
+            removed = list(set(aitems) - set(bitems))
+            added = list(set(bitems) - set(aitems))
+            out = '%s:%s%s' % (self.fieldname, ' removed "%s"' % ' '.join(removed) if removed else '', ' added "%s"' % ' '.join(added) if added else '')
+        elif self.fieldname in numeric_fields:
+            aval = int(self.oldvalue or 0)
+            bval = int(self.newvalue or 0)
+            if aval != 0:
+                percentchg = ((bval - aval) / float(aval)) * 100
+            else:
+                percentchg = 100
+            out = '%s changed from %s to %s (%s%d%%)' % (self.fieldname, self.oldvalue or "''", self.newvalue or "''", '+' if percentchg > 0 else '', percentchg)
+        elif self.fieldname in img_monitor_files:
+            if pathprefix:
+                prefix = 'Changes to %s ' % self.path
+            out = '(%s):\n  ' % self.fieldname
+            if self.filechanges:
+                out += '\n  '.join(['%s' % i for i in self.filechanges])
+            else:
+                alines = self.oldvalue.splitlines()
+                blines = self.newvalue.splitlines()
+                diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='')
+                out += '\n  '.join(list(diff))
+                out += '\n  --'
+        else:
+            out = '%s changed from "%s" to "%s"' % (self.fieldname, self.oldvalue, self.newvalue)
+
+        if self.related:
+            for chg in self.related:
+                for line in chg._str_internal(False).splitlines():
+                    out += '\n  * %s' % line
+
+        return '%s%s' % (prefix, out)
+
+class FileChange:
+    changetype_add = 'A'
+    changetype_remove = 'R'
+    changetype_type = 'T'
+    changetype_perms = 'P'
+    changetype_ownergroup = 'O'
+    changetype_link = 'L'
+
+    def __init__(self, path, changetype, oldvalue = None, newvalue = None):
+        self.path = path
+        self.changetype = changetype
+        self.oldvalue = oldvalue
+        self.newvalue = newvalue
+
+    def _ftype_str(self, ftype):
+        if ftype == '-':
+            return 'file'
+        elif ftype == 'd':
+            return 'directory'
+        elif ftype == 'l':
+            return 'symlink'
+        elif ftype == 'c':
+            return 'char device'
+        elif ftype == 'b':
+            return 'block device'
+        elif ftype == 'p':
+            return 'fifo'
+        elif ftype == 's':
+            return 'socket'
+        else:
+            return 'unknown (%s)' % ftype
+
+    def __str__(self):
+        if self.changetype == self.changetype_add:
+            return '%s was added' % self.path
+        elif self.changetype == self.changetype_remove:
+            return '%s was removed' % self.path
+        elif self.changetype == self.changetype_type:
+            return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue))
+        elif self.changetype == self.changetype_perms:
+            return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue)
+        elif self.changetype == self.changetype_ownergroup:
+            return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue)
+        elif self.changetype == self.changetype_link:
+            return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue)
+        else:
+            return '%s changed (unknown)' % self.path
+
+
+def blob_to_dict(blob):
+    alines = blob.data_stream.read().splitlines()
+    adict = {}
+    for line in alines:
+        splitv = [i.strip() for i in line.split('=',1)]
+        if splitv.count > 1:
+            adict[splitv[0]] = splitv[1]
+    return adict
+
+
+def file_list_to_dict(lines):
+    adict = {}
+    for line in lines:
+        # Leave the last few fields intact so we handle file names containing spaces
+        splitv = line.split(None,4)
+        # Grab the path and remove the leading .
+        path = splitv[4][1:].strip()
+        # Handle symlinks
+        if(' -> ' in path):
+            target = path.split(' -> ')[1]
+            path = path.split(' -> ')[0]
+            adict[path] = splitv[0:3] + [target]
+        else:
+            adict[path] = splitv[0:3]
+    return adict
+
+
+def compare_file_lists(alines, blines):
+    adict = file_list_to_dict(alines)
+    bdict = file_list_to_dict(blines)
+    filechanges = []
+    for path, splitv in adict.iteritems():
+        newsplitv = bdict.pop(path, None)
+        if newsplitv:
+            # Check type
+            oldvalue = splitv[0][0]
+            newvalue = newsplitv[0][0]
+            if oldvalue != newvalue:
+                filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue))
+            # Check permissions
+            oldvalue = splitv[0][1:]
+            newvalue = newsplitv[0][1:]
+            if oldvalue != newvalue:
+                filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue))
+            # Check owner/group
+            oldvalue = '%s/%s' % (splitv[1], splitv[2])
+            newvalue = '%s/%s' % (newsplitv[1], newsplitv[2])
+            if oldvalue != newvalue:
+                filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue))
+            # Check symlink target
+            if newsplitv[0][0] == 'l':
+                if splitv.count > 3:
+                    oldvalue = splitv[3]
+                else:
+                    oldvalue = None
+                newvalue = newsplitv[3]
+                if oldvalue != newvalue:
+                    filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue))
+        else:
+            filechanges.append(FileChange(path, FileChange.changetype_remove))
+
+    # Whatever is left over has been added
+    for path in bdict:
+        filechanges.append(FileChange(path, FileChange.changetype_add))
+
+    return filechanges
+
+
+def compare_lists(alines, blines):
+    removed = list(set(alines) - set(blines))
+    added = list(set(blines) - set(alines))
+
+    filechanges = []
+    for pkg in removed:
+        filechanges.append(FileChange(pkg, FileChange.changetype_remove))
+    for pkg in added:
+        filechanges.append(FileChange(pkg, FileChange.changetype_add))
+
+    return filechanges
+
+
+def compare_dict_blobs(path, ablob, bblob, report_all):
+    adict = blob_to_dict(ablob)
+    bdict = blob_to_dict(bblob)
+
+    changes = []
+    keys = list(set(adict.keys()) | set(bdict.keys()))
+    for key in keys:
+        astr = adict.get(key, '')
+        bstr = bdict.get(key, '')
+        if astr != bstr:
+            if (not report_all) and key in numeric_fields:
+                aval = int(astr or 0)
+                bval = int(bstr or 0)
+                if aval != 0:
+                    percentchg = ((bval - aval) / float(aval)) * 100
+                else:
+                    percentchg = 100
+                if percentchg < monitor_numeric_threshold:
+                    continue
+            chg = ChangeRecord(path, key, astr, bstr, key in monitor_fields)
+            changes.append(chg)
+    return changes
+
+
+def process_changes(repopath, revision1, revision2 = 'HEAD', report_all = False):
+    repo = git.Repo(repopath)
+    assert repo.bare == False
+    commit = repo.commit(revision1)
+    diff = commit.diff(revision2)
+
+    changes = []
+    for d in diff.iter_change_type('M'):
+        path = os.path.dirname(d.a_blob.path)
+        if path.startswith('packages/'):
+            changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all))
+        elif path.startswith('images/'):
+            filename = os.path.basename(d.a_blob.path)
+            if filename in img_monitor_files:
+                if filename == 'files-in-image.txt':
+                    alines = d.a_blob.data_stream.read().splitlines()
+                    blines = d.b_blob.data_stream.read().splitlines()
+                    filechanges = compare_file_lists(alines,blines)
+                    if filechanges:
+                        chg = ChangeRecord(path, filename, None, None, True)
+                        chg.filechanges = filechanges
+                        changes.append(chg)
+                elif filename == 'installed-package-names.txt':
+                    alines = d.a_blob.data_stream.read().splitlines()
+                    blines = d.b_blob.data_stream.read().splitlines()
+                    filechanges = compare_lists(alines,blines)
+                    if filechanges:
+                        chg = ChangeRecord(path, filename, None, None, True)
+                        chg.filechanges = filechanges
+                        changes.append(chg)
+                else:
+                    chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read(), True)
+                    changes.append(chg)
+            elif filename == 'image-info.txt':
+                changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all))
+
+    # Link related changes
+    for chg in changes:
+        if chg.monitored:
+            for chg2 in changes:
+                # (Check dirname in the case of fields from recipe info files)
+                if chg.path == chg2.path or os.path.dirname(chg.path) == chg2.path:
+                    if chg2.fieldname in related_fields.get(chg.fieldname, []):
+                        chg.related.append(chg2)
+                    elif chg.path.startswith('packages/') and chg2.fieldname in ['PE', 'PV', 'PR']:
+                        chg.related.append(chg2)
+
+    if report_all:
+        return changes
+    else:
+        return [chg for chg in changes if chg.monitored]

oe-core/meta/lib/oe/classextend.py

@@ -0,0 +1,81 @@
+class ClassExtender(object):
+    def __init__(self, extname, d):
+        self.extname = extname
+        self.d = d
+        self.pkgs_mapping = []
+
+    def extend_name(self, name):
+        if name.startswith("kernel-module"):
+            return name
+        if name.startswith("virtual/"):
+            subs = name.split("/", 1)[1]
+            if not subs.startswith(self.extname):
+                return "virtual/" + self.extname + "-" + subs
+            return name
+        if not name.startswith(self.extname):
+            return self.extname + "-" + name
+        return name
+
+    def map_variable(self, varname, setvar = True):
+        var = self.d.getVar(varname, True)
+        if not var:
+            return ""
+        var = var.split()
+        newvar = []
+        for v in var:
+            newvar.append(self.extend_name(v))
+        newdata =  " ".join(newvar)
+        if setvar:
+            self.d.setVar(varname, newdata)
+        return newdata
+
+    def map_depends(self, dep):
+        if dep.endswith(("-native", "-native-runtime")):
+            return dep
+        else:
+            return self.extend_name(dep)
+
+    def map_depends_variable(self, varname, suffix = ""):
+        if suffix:
+            varname = varname + "_" + suffix
+        deps = self.d.getVar(varname, True)
+        if not deps:
+            return
+        deps = bb.utils.explode_deps(deps)
+        newdeps = []
+        for dep in deps:
+            newdeps.append(self.map_depends(dep))
+        self.d.setVar(varname, " ".join(newdeps))
+
+    def map_packagevars(self):
+        for pkg in (self.d.getVar("PACKAGES", True).split() + [""]):
+            self.map_depends_variable("RDEPENDS", pkg)
+            self.map_depends_variable("RRECOMMENDS", pkg)
+            self.map_depends_variable("RSUGGESTS", pkg)
+            self.map_depends_variable("RPROVIDES", pkg)
+            self.map_depends_variable("RREPLACES", pkg)
+            self.map_depends_variable("RCONFLICTS", pkg)
+            self.map_depends_variable("PKG", pkg)
+
+    def rename_packages(self):
+        for pkg in (self.d.getVar("PACKAGES", True) or "").split():
+            if pkg.startswith(self.extname):
+               self.pkgs_mapping.append([pkg.split(self.extname + "-")[1], pkg])
+               continue
+            self.pkgs_mapping.append([pkg, self.extend_name(pkg)])
+
+        self.d.setVar("PACKAGES", " ".join([row[1] for row in self.pkgs_mapping]))
+
+    def rename_package_variables(self, variables):
+        for pkg_mapping in self.pkgs_mapping:
+            for subs in variables:
+                self.d.renameVar("%s_%s" % (subs, pkg_mapping[0]), "%s_%s" % (subs, pkg_mapping[1]))
+
+class NativesdkClassExtender(ClassExtender):
+    def map_depends(self, dep):
+        if dep.endswith(("-native", "-native-runtime", "-cross")):
+            return dep
+        elif dep.endswith(("-gcc-intermediate", "-gcc-initial", "-gcc", "-g++")):
+            return dep + "-crosssdk"
+        else:
+            return self.extend_name(dep)

oe-core/meta/lib/oe/classutils.py

@@ -0,0 +1,43 @@
+class ClassRegistry(type):
+    """Maintain a registry of classes, indexed by name.
+
+Note that this implementation requires that the names be unique, as it uses
+a dictionary to hold the classes by name.
+
+The name in the registry can be overridden via the 'name' attribute of the
+class, and the 'priority' attribute controls priority. The prioritized()
+method returns the registered classes in priority order.
+
+Subclasses of ClassRegistry may define an 'implemented' property to exert
+control over whether the class will be added to the registry (e.g. to keep
+abstract base classes out of the registry)."""
+    priority = 0
+    class __metaclass__(type):
+        """Give each ClassRegistry their own registry"""
+        def __init__(cls, name, bases, attrs):
+            cls.registry = {}
+            type.__init__(cls, name, bases, attrs)
+
+    def __init__(cls, name, bases, attrs):
+        super(ClassRegistry, cls).__init__(name, bases, attrs)
+        try:
+            if not cls.implemented:
+                return
+        except AttributeError:
+            pass
+
+        try:
+            cls.name
+        except AttributeError:
+            cls.name = name
+        cls.registry[cls.name] = cls
+
+    @classmethod
+    def prioritized(tcls):
+        return sorted(tcls.registry.values(),
+                      key=lambda v: v.priority, reverse=True)
+
+    def unregister(cls):
+        for key in cls.registry.keys():
+            if cls.registry[key] is cls:
+                del cls.registry[key]

oe-core/meta/lib/oe/data.py

@@ -0,0 +1,18 @@
+import oe.maketype
+import bb.msg
+
+def typed_value(key, d):
+    """Construct a value for the specified metadata variable, using its flags
+    to determine the type and parameters for construction."""
+    var_type = d.getVarFlag(key, 'type')
+    flags = d.getVarFlags(key)
+    if flags is not None:
+        flags = dict((flag, bb.data.expand(value, d))
+                     for flag, value in flags.iteritems())
+    else:
+        flags = {}
+
+    try:
+        return oe.maketype.create(d.getVar(key, True) or '', var_type, **flags)
+    except (TypeError, ValueError), exc:
+        bb.msg.fatal("Data", "%s: %s" % (key, str(exc)))

oe-core/meta/lib/oe/distro_check.py

@@ -0,0 +1,377 @@
+def get_links_from_url(url):
+    "Return all the href links found on the web location"
+
+    import urllib, sgmllib
+    
+    class LinksParser(sgmllib.SGMLParser):
+        def parse(self, s):
+            "Parse the given string 's'."
+            self.feed(s)
+            self.close()
+    
+        def __init__(self, verbose=0):
+            "Initialise an object passing 'verbose' to the superclass."
+            sgmllib.SGMLParser.__init__(self, verbose)
+            self.hyperlinks = []
+    
+        def start_a(self, attributes):
+            "Process a hyperlink and its 'attributes'."
+            for name, value in attributes:
+                if name == "href":
+                    self.hyperlinks.append(value.strip('/'))
+    
+        def get_hyperlinks(self):
+            "Return the list of hyperlinks."
+            return self.hyperlinks
+
+    sock = urllib.urlopen(url)
+    webpage = sock.read()
+    sock.close()
+
+    linksparser = LinksParser()
+    linksparser.parse(webpage)
+    return linksparser.get_hyperlinks()
+
+def find_latest_numeric_release(url):
+    "Find the latest listed numeric release on the given url"
+    max=0
+    maxstr=""
+    for link in get_links_from_url(url):
+        try:
+            release = float(link)
+        except:
+            release = 0
+        if release > max:
+            max = release
+            maxstr = link
+    return maxstr
+
+def is_src_rpm(name):
+    "Check if the link is pointing to a src.rpm file"
+    if name[-8:] == ".src.rpm":
+        return True
+    else:
+        return False
+
+def package_name_from_srpm(srpm):
+    "Strip out the package name from the src.rpm filename"
+    strings = srpm.split('-')
+    package_name = strings[0]
+    for i in range(1, len (strings) - 1):
+        str = strings[i]
+        if not str[0].isdigit():
+            package_name += '-' + str
+    return package_name
+
+def clean_package_list(package_list):
+    "Removes multiple entries of packages and sorts the list"
+    set = {}
+    map(set.__setitem__, package_list, [])
+    return set.keys()
+
+
+def get_latest_released_meego_source_package_list():
+    "Returns list of all the name os packages in the latest meego distro"
+
+    if not os.path.isfile("/tmp/Meego-1.1"):
+        os.mknod("/tmp/Meego-1.1")
+    f = open("/tmp/Meego-1.1", "r")
+    package_names = []
+    for line in f:
+        package_names.append(line[:-1] + ":" + "main") # Also strip the '\n' at the end
+     
+    package_list=clean_package_list(package_names)
+    return "1.0", package_list
+
+def get_source_package_list_from_url(url, section):
+    "Return a sectioned list of package names from a URL list"
+
+    bb.note("Reading %s: %s" % (url, section))
+    links = get_links_from_url(url)
+    srpms = filter(is_src_rpm, links)
+    names_list = map(package_name_from_srpm, srpms)
+
+    new_pkgs = []
+    for pkgs in names_list:
+       new_pkgs.append(pkgs + ":" + section)
+
+    return new_pkgs
+
+def get_latest_released_fedora_source_package_list():
+    "Returns list of all the name os packages in the latest fedora distro"
+    latest = find_latest_numeric_release("http://download.fedora.redhat.com/pub/fedora/linux/releases/")
+
+    package_names = get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main")
+
+#    package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything")
+    package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates")
+
+    package_list=clean_package_list(package_names)
+        
+    return latest, package_list
+
+def get_latest_released_opensuse_source_package_list():
+    "Returns list of all the name os packages in the latest opensuse distro"
+    latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
+
+    package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main")
+    package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates")
+
+    package_list=clean_package_list(package_names)
+    return latest, package_list
+
+def get_latest_released_mandriva_source_package_list():
+    "Returns list of all the name os packages in the latest mandriva distro"
+    latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
+    package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main")
+#    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib")
+    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates")
+
+    package_list=clean_package_list(package_names)
+    return latest, package_list
+
+def find_latest_debian_release(url):
+    "Find the latest listed debian release on the given url"
+
+    releases = []
+    for link in get_links_from_url(url):
+        if link[:6] == "Debian":
+            if ';' not in link:
+                releases.append(link)
+    releases.sort()
+    try:
+        return releases.pop()[6:]
+    except:
+        return "_NotFound_"
+
+def get_debian_style_source_package_list(url, section):
+    "Return the list of package-names stored in the debian style Sources.gz file"
+    import urllib
+    sock = urllib.urlopen(url)
+    import tempfile
+    tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
+    tmpfilename=tmpfile.name
+    tmpfile.write(sock.read())
+    sock.close()
+    tmpfile.close()
+    import gzip
+    bb.note("Reading %s: %s" % (url, section))
+
+    f = gzip.open(tmpfilename)
+    package_names = []
+    for line in f:
+        if line[:9] == "Package: ":
+            package_names.append(line[9:-1] + ":" + section) # Also strip the '\n' at the end
+    os.unlink(tmpfilename)
+
+    return package_names
+
+def get_latest_released_debian_source_package_list():
+    "Returns list of all the name os packages in the latest debian distro"
+    latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/")
+    url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" 
+    package_names = get_debian_style_source_package_list(url, "main")
+#    url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" 
+#    package_names += get_debian_style_source_package_list(url, "contrib")
+    url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" 
+    package_names += get_debian_style_source_package_list(url, "updates")
+    package_list=clean_package_list(package_names)
+    return latest, package_list
+
+def find_latest_ubuntu_release(url):
+    "Find the latest listed ubuntu release on the given url"
+    url += "?C=M;O=D" # Descending Sort by Last Modified
+    for link in get_links_from_url(url):
+        if link[-8:] == "-updates":
+            return link[:-8]
+    return "_NotFound_"
+
+def get_latest_released_ubuntu_source_package_list():
+    "Returns list of all the name os packages in the latest ubuntu distro"
+    latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
+    url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
+    package_names = get_debian_style_source_package_list(url, "main")
+#    url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest
+#    package_names += get_debian_style_source_package_list(url, "multiverse")
+#    url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest
+#    package_names += get_debian_style_source_package_list(url, "universe")
+    url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
+    package_names += get_debian_style_source_package_list(url, "updates")
+    package_list=clean_package_list(package_names)
+    return latest, package_list
+
+def create_distro_packages_list(distro_check_dir):
+    pkglst_dir = os.path.join(distro_check_dir, "package_lists")
+    if not os.path.isdir (pkglst_dir):
+        os.makedirs(pkglst_dir)
+    # first clear old stuff
+    for file in os.listdir(pkglst_dir):
+        os.unlink(os.path.join(pkglst_dir, file))
+ 
+    per_distro_functions = [
+                            ["Debian", get_latest_released_debian_source_package_list],
+                            ["Ubuntu", get_latest_released_ubuntu_source_package_list],
+                            ["Fedora", get_latest_released_fedora_source_package_list],
+                            ["OpenSuSE", get_latest_released_opensuse_source_package_list],
+                            ["Mandriva", get_latest_released_mandriva_source_package_list],
+                            ["Meego", get_latest_released_meego_source_package_list]
+                           ]
+ 
+    from datetime import datetime
+    begin = datetime.now()
+    for distro in per_distro_functions:
+        name = distro[0]
+        release, package_list = distro[1]()
+        bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
+        package_list_file = os.path.join(pkglst_dir, name + "-" + release)
+        f = open(package_list_file, "w+b")
+        for pkg in package_list:
+            f.write(pkg + "\n")
+        f.close()
+    end = datetime.now()
+    delta = end - begin
+    bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
+
+def update_distro_data(distro_check_dir, datetime):
+    """
+        If distro packages list data is old then rebuild it.
+        The operations has to be protected by a lock so that
+        only one thread performes it at a time.
+    """
+    if not os.path.isdir (distro_check_dir):
+        try:
+            bb.note ("Making new directory: %s" % distro_check_dir)
+            os.makedirs (distro_check_dir)
+        except OSError:
+            raise Exception('Unable to create directory %s' % (distro_check_dir))
+
+
+    datetime_file = os.path.join(distro_check_dir, "build_datetime")
+    saved_datetime = "_invalid_"
+    import fcntl
+    try:
+        if not os.path.exists(datetime_file):
+            open(datetime_file, 'w+b').close() # touch the file so that the next open won't fail
+
+        f = open(datetime_file, "r+b")
+        fcntl.lockf(f, fcntl.LOCK_EX)
+        saved_datetime = f.read()
+        if saved_datetime[0:8] != datetime[0:8]:
+            bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
+            bb.note("Regenerating distro package lists")
+            create_distro_packages_list(distro_check_dir)
+            f.seek(0)
+            f.write(datetime)
+
+    except OSError:
+        raise Exception('Unable to read/write this file: %s' % (datetime_file))
+    finally:
+        fcntl.lockf(f, fcntl.LOCK_UN)
+        f.close()
+ 
+def compare_in_distro_packages_list(distro_check_dir, d):
+    if not os.path.isdir(distro_check_dir):
+        raise Exception("compare_in_distro_packages_list: invalid distro_check_dir passed")
+        
+    localdata = bb.data.createCopy(d)
+    pkglst_dir = os.path.join(distro_check_dir, "package_lists")
+    matching_distros = []
+    pn = d.getVar('PN', True)
+    recipe_name = d.getVar('PN', True)
+    bb.note("Checking: %s" % pn)
+
+    trim_dict = dict({"-native":"-native", "-cross":"-cross", "-initial":"-initial"})
+
+    if pn.find("-native") != -1:
+        pnstripped = pn.split("-native")
+        localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
+        bb.data.update_data(localdata)
+        recipe_name = pnstripped[0]
+
+    if pn.find("-cross") != -1:
+        pnstripped = pn.split("-cross")
+        localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
+        bb.data.update_data(localdata)
+        recipe_name = pnstripped[0]
+
+    if pn.find("-initial") != -1:
+        pnstripped = pn.split("-initial")
+        localdata.setVar('OVERRIDES', "pn-" + pnstripped[0] + ":" + d.getVar('OVERRIDES', True))
+        bb.data.update_data(localdata)
+        recipe_name = pnstripped[0]
+
+    bb.note("Recipe: %s" % recipe_name)
+    tmp = localdata.getVar('DISTRO_PN_ALIAS', True)
+
+    distro_exceptions = dict({"OE-Core":'OE-Core', "OpenedHand":'OpenedHand', "Intel":'Intel', "Upstream":'Upstream', "Windriver":'Windriver', "OSPDT":'OSPDT Approved', "Poky":'poky'})
+
+    if tmp:
+        list = tmp.split(' ')
+        for str in list:
+            if str and str.find("=") == -1 and distro_exceptions[str]:
+                matching_distros.append(str)
+
+    distro_pn_aliases = {}
+    if tmp:
+        list = tmp.split(' ')
+        for str in list:
+            if str.find("=") != -1:
+                (dist, pn_alias) = str.split('=')
+                distro_pn_aliases[dist.strip().lower()] = pn_alias.strip()
+ 
+    for file in os.listdir(pkglst_dir):
+        (distro, distro_release) = file.split("-")
+        f = open(os.path.join(pkglst_dir, file), "rb")
+        for line in f:
+            (pkg, section) = line.split(":")
+            if distro.lower() in distro_pn_aliases:
+                pn = distro_pn_aliases[distro.lower()]
+            else:
+                pn = recipe_name
+            if pn == pkg:
+                matching_distros.append(distro + "-" + section[:-1]) # strip the \n at the end
+                f.close()
+                break
+        f.close()
+
+    
+    if tmp != None:
+	list = tmp.split(' ')
+	for item in list:
+            matching_distros.append(item)
+    bb.note("Matching: %s" % matching_distros)
+    return matching_distros
+
+def create_log_file(d, logname):
+    logpath = d.getVar('LOG_DIR', True)
+    bb.utils.mkdirhier(logpath)
+    logfn, logsuffix = os.path.splitext(logname)
+    logfile = os.path.join(logpath, "%s.%s%s" % (logfn, d.getVar('DATETIME', True), logsuffix))
+    if not os.path.exists(logfile):
+            slogfile = os.path.join(logpath, logname)
+            if os.path.exists(slogfile):
+                    os.remove(slogfile)
+            os.system("touch %s" % logfile)
+            os.symlink(logfile, slogfile)
+            d.setVar('LOG_FILE', logfile)
+    return logfile
+
+
+def save_distro_check_result(result, datetime, result_file, d):
+    pn = d.getVar('PN', True)
+    logdir = d.getVar('LOG_DIR', True)
+    if not logdir:
+        bb.error("LOG_DIR variable is not defined, can't write the distro_check results")
+        return
+    if not os.path.isdir(logdir):
+        os.makedirs(logdir)
+    line = pn
+    for i in result:
+        line = line + "," + i
+    f = open(result_file, "a")
+    import fcntl
+    fcntl.lockf(f, fcntl.LOCK_EX)
+    f.seek(0, os.SEEK_END) # seek to the end of file
+    f.write(line + "\n")
+    fcntl.lockf(f, fcntl.LOCK_UN)
+    f.close()

oe-core/meta/lib/oe/license.py

@@ -0,0 +1,113 @@
+# vi:sts=4:sw=4:et
+"""Code for parsing OpenEmbedded license strings"""
+
+import ast
+import re
+from fnmatch import fnmatchcase as fnmatch
+
+class LicenseError(StandardError):
+    pass
+
+class LicenseSyntaxError(LicenseError):
+    def __init__(self, licensestr, exc):
+        self.licensestr = licensestr
+        self.exc = exc
+        LicenseError.__init__(self)
+
+    def __str__(self):
+        return "error in '%s': %s" % (self.licensestr, self.exc)
+
+class InvalidLicense(LicenseError):
+    def __init__(self, license):
+        self.license = license
+        LicenseError.__init__(self)
+
+    def __str__(self):
+        return "invalid characters in license '%s'" % self.license
+
+license_operator = re.compile('([&|() ])')
+license_pattern = re.compile('[a-zA-Z0-9.+_\-]+$')
+
+class LicenseVisitor(ast.NodeVisitor):
+    """Syntax tree visitor which can accept OpenEmbedded license strings"""
+    def visit_string(self, licensestr):
+        new_elements = []
+        elements = filter(lambda x: x.strip(), license_operator.split(licensestr))
+        for pos, element in enumerate(elements):
+            if license_pattern.match(element):
+                if pos > 0 and license_pattern.match(elements[pos-1]):
+                    new_elements.append('&')
+                element = '"' + element + '"'
+            elif not license_operator.match(element):
+                raise InvalidLicense(element)
+            new_elements.append(element)
+
+        self.visit(ast.parse(' '.join(new_elements)))
+
+class FlattenVisitor(LicenseVisitor):
+    """Flatten a license tree (parsed from a string) by selecting one of each
+    set of OR options, in the way the user specifies"""
+    def __init__(self, choose_licenses):
+        self.choose_licenses = choose_licenses
+        self.licenses = []
+        LicenseVisitor.__init__(self)
+
+    def visit_Str(self, node):
+        self.licenses.append(node.s)
+
+    def visit_BinOp(self, node):
+        if isinstance(node.op, ast.BitOr):
+            left = FlattenVisitor(self.choose_licenses)
+            left.visit(node.left)
+
+            right = FlattenVisitor(self.choose_licenses)
+            right.visit(node.right)
+
+            selected = self.choose_licenses(left.licenses, right.licenses)
+            self.licenses.extend(selected)
+        else:
+            self.generic_visit(node)
+
+def flattened_licenses(licensestr, choose_licenses):
+    """Given a license string and choose_licenses function, return a flat list of licenses"""
+    flatten = FlattenVisitor(choose_licenses)
+    try:
+        flatten.visit_string(licensestr)
+    except SyntaxError as exc:
+        raise LicenseSyntaxError(licensestr, exc)
+    return flatten.licenses
+
+def is_included(licensestr, whitelist=None, blacklist=None):
+    """Given a license string and whitelist and blacklist, determine if the
+    license string matches the whitelist and does not match the blacklist.
+
+    Returns a tuple holding the boolean state and a list of the applicable
+    licenses which were excluded (or None, if the state is True)
+    """
+
+    def include_license(license):
+        return (any(fnmatch(license, pattern) for pattern in whitelist) and not
+                any(fnmatch(license, pattern) for pattern in blacklist))
+
+    def choose_licenses(alpha, beta):
+        """Select the option in an OR which is the 'best' (has the most
+        included licenses)."""
+        alpha_weight = len(filter(include_license, alpha))
+        beta_weight = len(filter(include_license, beta))
+        if alpha_weight > beta_weight:
+            return alpha
+        else:
+            return beta
+
+    if not whitelist:
+        whitelist = ['*']
+
+    if not blacklist:
+        blacklist = []
+
+    licenses = flattened_licenses(licensestr, choose_licenses)
+    excluded = filter(lambda lic: not include_license(lic), licenses)
+    if excluded:
+        return False, excluded
+    else:
+        return True, None

oe-core/meta/lib/oe/maketype.py

@@ -0,0 +1,100 @@
+"""OpenEmbedded variable typing support
+
+Types are defined in the metadata by name, using the 'type' flag on a
+variable.  Other flags may be utilized in the construction of the types.  See
+the arguments of the type's factory for details.
+"""
+
+import bb
+import inspect
+import types
+
+available_types = {}
+
+class MissingFlag(TypeError):
+    """A particular flag is required to construct the type, but has not been
+    provided."""
+    def __init__(self, flag, type):
+        self.flag = flag
+        self.type = type
+        TypeError.__init__(self)
+
+    def __str__(self):
+        return "Type '%s' requires flag '%s'" % (self.type, self.flag)
+
+def factory(var_type):
+    """Return the factory for a specified type."""
+    if var_type is None:
+        raise TypeError("No type specified. Valid types: %s" %
+                        ', '.join(available_types))
+    try:
+        return available_types[var_type]
+    except KeyError:
+        raise TypeError("Invalid type '%s':\n  Valid types: %s" %
+                        (var_type, ', '.join(available_types)))
+
+def create(value, var_type, **flags):
+    """Create an object of the specified type, given the specified flags and
+    string value."""
+    obj = factory(var_type)
+    objflags = {}
+    for flag in obj.flags:
+        if flag not in flags:
+            if flag not in obj.optflags:
+                raise MissingFlag(flag, var_type)
+        else:
+            objflags[flag] = flags[flag]
+
+    return obj(value, **objflags)
+
+def get_callable_args(obj):
+    """Grab all but the first argument of the specified callable, returning
+    the list, as well as a list of which of the arguments have default
+    values."""
+    if type(obj) is type:
+        obj = obj.__init__
+
+    args, varargs, keywords, defaults = inspect.getargspec(obj)
+    flaglist = []
+    if args:
+        if len(args) > 1 and args[0] == 'self':
+            args = args[1:]
+        flaglist.extend(args)
+
+    optional = set()
+    if defaults:
+        optional |= set(flaglist[-len(defaults):])
+    return flaglist, optional
+
+def factory_setup(name, obj):
+    """Prepare a factory for use."""
+    args, optional = get_callable_args(obj)
+    extra_args = args[1:]
+    if extra_args:
+        obj.flags, optional = extra_args, optional
+        obj.optflags = set(optional)
+    else:
+        obj.flags = obj.optflags = ()
+
+    if not hasattr(obj, 'name'):
+        obj.name = name
+
+def register(name, factory):
+    """Register a type, given its name and a factory callable.
+
+    Determines the required and optional flags from the factory's
+    arguments."""
+    factory_setup(name, factory)
+    available_types[factory.name] = factory
+
+
+# Register all our included types
+for name in dir(types):
+    if name.startswith('_'):
+        continue
+
+    obj = getattr(types, name)
+    if not callable(obj):
+        continue
+
+    register(name, obj)

oe-core/meta/lib/oe/packagedata.py

@@ -0,0 +1,109 @@
+import os
+import bb.data
+import codecs
+
+def packaged(pkg, d):
+    return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK)
+
+def read_pkgdatafile(fn):
+    pkgdata = {}
+
+    def decode(str):
+        c = codecs.getdecoder("string_escape")
+        return c(str)[0]
+
+    if os.access(fn, os.R_OK):
+        import re
+        f = file(fn, 'r')
+        lines = f.readlines()
+        f.close()
+        r = re.compile("([^:]+):\s*(.*)")
+        for l in lines:
+            m = r.match(l)
+            if m:
+                pkgdata[m.group(1)] = decode(m.group(2))
+
+    return pkgdata
+
+def get_subpkgedata_fn(pkg, d):
+    archs = bb.data.expand("${PACKAGE_ARCHS}", d).split(" ")
+    archs.reverse()
+    pkgdata = bb.data.expand('${TMPDIR}/pkgdata/', d)
+    targetdir = bb.data.expand('${TARGET_VENDOR}-${TARGET_OS}/runtime/', d)
+    for arch in archs:
+        fn = pkgdata + arch + targetdir + pkg
+        if os.path.exists(fn):
+            return fn
+    return bb.data.expand('${PKGDATA_DIR}/runtime/%s' % pkg, d)
+
+def has_subpkgdata(pkg, d):
+    return os.access(get_subpkgedata_fn(pkg, d), os.R_OK)
+
+def read_subpkgdata(pkg, d):
+    return read_pkgdatafile(get_subpkgedata_fn(pkg, d))
+
+def has_pkgdata(pn, d):
+    fn = bb.data.expand('${PKGDATA_DIR}/%s' % pn, d)
+    return os.access(fn, os.R_OK)
+
+def read_pkgdata(pn, d):
+    fn = bb.data.expand('${PKGDATA_DIR}/%s' % pn, d)
+    return read_pkgdatafile(fn)
+
+#
+# Collapse FOO_pkg variables into FOO
+#
+def read_subpkgdata_dict(pkg, d):
+    ret = {}
+    subd = read_pkgdatafile(get_subpkgedata_fn(pkg, d))
+    for var in subd:
+        newvar = var.replace("_" + pkg, "")
+        if newvar == var and var + "_" + pkg in subd:
+            continue
+        ret[newvar] = subd[var]
+    return ret
+
+def _pkgmap(d):
+    """Return a dictionary mapping package to recipe name."""
+
+    target_os = d.getVar("TARGET_OS", True)
+    target_vendor = d.getVar("TARGET_VENDOR", True)
+    basedir = os.path.dirname(d.getVar("PKGDATA_DIR", True))
+
+    dirs = ("%s%s-%s" % (arch, target_vendor, target_os)
+            for arch in d.getVar("PACKAGE_ARCHS", True).split())
+
+    pkgmap = {}
+    for pkgdatadir in (os.path.join(basedir, sys) for sys in dirs):
+        try:
+            files = os.listdir(pkgdatadir)
+        except OSError:
+            continue
+
+        for pn in filter(lambda f: not os.path.isdir(os.path.join(pkgdatadir, f)), files):
+            try:
+                pkgdata = read_pkgdatafile(os.path.join(pkgdatadir, pn))
+            except OSError:
+                continue
+
+            packages = pkgdata.get("PACKAGES") or ""
+            for pkg in packages.split():
+                pkgmap[pkg] = pn
+
+    return pkgmap
+
+def pkgmap(d):
+    """Return a dictionary mapping package to recipe name.
+    Cache the mapping in the metadata"""
+
+    pkgmap_data = d.getVar("__pkgmap_data", False)
+    if pkgmap_data is None:
+        pkgmap_data = _pkgmap(d)
+        d.setVar("__pkgmap_data", pkgmap_data)
+
+    return pkgmap_data
+
+def recipename(pkg, d):
+    """Return the recipe name for the given binary package name."""
+
+    return pkgmap(d).get(pkg)

oe-core/meta/lib/oe/packagegroup.py

@@ -0,0 +1,29 @@
+import itertools
+
+def is_optional(group, d):
+    return bool(d.getVarFlag("PACKAGE_GROUP_%s" % group, "optional"))
+
+def packages(groups, d):
+    for group in groups:
+        for pkg in (d.getVar("PACKAGE_GROUP_%s" % group, True) or "").split():
+            yield pkg
+
+def required_packages(groups, d):
+    req = filter(lambda group: not is_optional(group, d), groups)
+    return packages(req, d)
+
+def optional_packages(groups, d):
+    opt = filter(lambda group: is_optional(group, d), groups)
+    return packages(opt, d)
+
+def active_packages(features, d):
+    return itertools.chain(required_packages(features, d),
+                           optional_packages(features, d))
+
+def active_recipes(features, d):
+    import oe.packagedata
+
+    for pkg in active_packages(features, d):
+        recipe = oe.packagedata.recipename(pkg, d)
+        if recipe:
+            yield recipe

oe-core/meta/lib/oe/patch.py

@@ -0,0 +1,402 @@
+import oe.path
+import os
+import bb.utils, bb.msg, bb.data, bb.fetch2
+
+class NotFoundError(bb.BBHandledException):
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self):
+        return "Error: %s not found." % self.path
+
+class CmdError(bb.BBHandledException):
+    def __init__(self, exitstatus, output):
+        self.status = exitstatus
+        self.output = output
+
+    def __str__(self):
+        return "Command Error: exit status: %d  Output:\n%s" % (self.status, self.output)
+
+
+def runcmd(args, dir = None):
+    import commands
+
+    if dir:
+        olddir = os.path.abspath(os.curdir)
+        if not os.path.exists(dir):
+            raise NotFoundError(dir)
+        os.chdir(dir)
+        # print("cwd: %s -> %s" % (olddir, dir))
+
+    try:
+        args = [ commands.mkarg(str(arg)) for arg in args ]
+        cmd = " ".join(args)
+        # print("cmd: %s" % cmd)
+        (exitstatus, output) = commands.getstatusoutput(cmd)
+        if exitstatus != 0:
+            raise CmdError(exitstatus >> 8, output)
+        return output
+
+    finally:
+        if dir:
+            os.chdir(olddir)
+
+class PatchError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+
+    def __str__(self):
+        return "Patch Error: %s" % self.msg
+
+class PatchSet(object):
+    defaults = {
+        "strippath": 1
+    }
+
+    def __init__(self, dir, d):
+        self.dir = dir
+        self.d = d
+        self.patches = []
+        self._current = None
+
+    def current(self):
+        return self._current
+
+    def Clean(self):
+        """
+        Clean out the patch set.  Generally includes unapplying all
+        patches and wiping out all associated metadata.
+        """
+        raise NotImplementedError()
+
+    def Import(self, patch, force):
+        if not patch.get("file"):
+            if not patch.get("remote"):
+                raise PatchError("Patch file must be specified in patch import.")
+            else:
+                patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
+
+        for param in PatchSet.defaults:
+            if not patch.get(param):
+                patch[param] = PatchSet.defaults[param]
+
+        if patch.get("remote"):
+            patch["file"] = bb.data.expand(bb.fetch2.localpath(patch["remote"], self.d), self.d)
+
+        patch["filemd5"] = bb.utils.md5_file(patch["file"])
+
+    def Push(self, force):
+        raise NotImplementedError()
+
+    def Pop(self, force):
+        raise NotImplementedError()
+
+    def Refresh(self, remote = None, all = None):
+        raise NotImplementedError()
+
+
+class PatchTree(PatchSet):
+    def __init__(self, dir, d):
+        PatchSet.__init__(self, dir, d)
+
+    def Import(self, patch, force = None):
+        """"""
+        PatchSet.Import(self, patch, force)
+
+        if self._current is not None:
+            i = self._current + 1
+        else:
+            i = 0
+        self.patches.insert(i, patch)
+
+    def _applypatch(self, patch, force = False, reverse = False, run = True):
+        shellcmd = ["cat", patch['file'], "|", "patch", "-p", patch['strippath']]
+        if reverse:
+            shellcmd.append('-R')
+
+        if not run:
+            return "sh" + "-c" + " ".join(shellcmd)
+
+        if not force:
+            shellcmd.append('--dry-run')
+
+        output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+
+        if force:
+            return
+
+        shellcmd.pop(len(shellcmd) - 1)
+        output = runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+        return output
+
+    def Push(self, force = False, all = False, run = True):
+        bb.note("self._current is %s" % self._current)
+        bb.note("patches is %s" % self.patches)
+        if all:
+            for i in self.patches:
+                if self._current is not None:
+                    self._current = self._current + 1
+                else:
+                    self._current = 0
+                bb.note("applying patch %s" % i)
+                self._applypatch(i, force)
+        else:
+            if self._current is not None:
+                self._current = self._current + 1
+            else:
+                self._current = 0
+            bb.note("applying patch %s" % self.patches[self._current])
+            return self._applypatch(self.patches[self._current], force)
+
+
+    def Pop(self, force = None, all = None):
+        if all:
+            for i in self.patches:
+                self._applypatch(i, force, True)
+        else:
+            self._applypatch(self.patches[self._current], force, True)
+
+    def Clean(self):
+        """"""
+
+class GitApplyTree(PatchTree):
+    def __init__(self, dir, d):
+        PatchTree.__init__(self, dir, d)
+
+    def _applypatch(self, patch, force = False, reverse = False, run = True):
+        shellcmd = ["git", "--git-dir=.", "apply", "-p%s" % patch['strippath']]
+
+        if reverse:
+            shellcmd.append('-R')
+
+        shellcmd.append(patch['file'])
+
+        if not run:
+            return "sh" + "-c" + " ".join(shellcmd)
+
+        return runcmd(["sh", "-c", " ".join(shellcmd)], self.dir)
+
+
+class QuiltTree(PatchSet):
+    def _runcmd(self, args, run = True):
+        quiltrc = self.d.getVar('QUILTRCFILE', 1)
+        if not run:
+            return ["quilt"] + ["--quiltrc"] + [quiltrc] + args
+        runcmd(["quilt"] + ["--quiltrc"] + [quiltrc] + args, self.dir)
+
+    def _quiltpatchpath(self, file):
+        return os.path.join(self.dir, "patches", os.path.basename(file))
+
+
+    def __init__(self, dir, d):
+        PatchSet.__init__(self, dir, d)
+        self.initialized = False
+        p = os.path.join(self.dir, 'patches')
+        if not os.path.exists(p):
+            os.makedirs(p)
+
+    def Clean(self):
+        try:
+            self._runcmd(["pop", "-a", "-f"])
+            oe.path.remove(os.path.join(self.dir, "patches","series"))
+        except Exception:
+            pass
+        self.initialized = True
+
+    def InitFromDir(self):
+        # read series -> self.patches
+        seriespath = os.path.join(self.dir, 'patches', 'series')
+        if not os.path.exists(self.dir):
+            raise NotFoundError(self.dir)
+        if os.path.exists(seriespath):
+            series = file(seriespath, 'r')
+            for line in series.readlines():
+                patch = {}
+                parts = line.strip().split()
+                patch["quiltfile"] = self._quiltpatchpath(parts[0])
+                patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"])
+                if len(parts) > 1:
+                    patch["strippath"] = parts[1][2:]
+                self.patches.append(patch)
+            series.close()
+
+            # determine which patches are applied -> self._current
+            try:
+                output = runcmd(["quilt", "applied"], self.dir)
+            except CmdError:
+                import sys
+                if sys.exc_value.output.strip() == "No patches applied":
+                    return
+                else:
+                    raise
+            output = [val for val in output.split('\n') if not val.startswith('#')]
+            for patch in self.patches:
+                if os.path.basename(patch["quiltfile"]) == output[-1]:
+                    self._current = self.patches.index(patch)
+        self.initialized = True
+
+    def Import(self, patch, force = None):
+        if not self.initialized:
+            self.InitFromDir()
+        PatchSet.Import(self, patch, force)
+        oe.path.symlink(patch["file"], self._quiltpatchpath(patch["file"]))
+        f = open(os.path.join(self.dir, "patches","series"), "a");
+        f.write(os.path.basename(patch["file"]) + " -p" + patch["strippath"]+"\n")
+        f.close()
+        patch["quiltfile"] = self._quiltpatchpath(patch["file"])
+        patch["quiltfilemd5"] = bb.utils.md5_file(patch["quiltfile"])
+
+        # TODO: determine if the file being imported:
+        #      1) is already imported, and is the same
+        #      2) is already imported, but differs
+
+        self.patches.insert(self._current or 0, patch)
+
+
+    def Push(self, force = False, all = False, run = True):
+        # quilt push [-f]
+
+        args = ["push"]
+        if force:
+            args.append("-f")
+        if all:
+            args.append("-a")
+        if not run:
+            return self._runcmd(args, run)
+
+        self._runcmd(args)
+
+        if self._current is not None:
+            self._current = self._current + 1
+        else:
+            self._current = 0
+
+    def Pop(self, force = None, all = None):
+        # quilt pop [-f]
+        args = ["pop"]
+        if force:
+            args.append("-f")
+        if all:
+            args.append("-a")
+
+        self._runcmd(args)
+
+        if self._current == 0:
+            self._current = None
+
+        if self._current is not None:
+            self._current = self._current - 1
+
+    def Refresh(self, **kwargs):
+        if kwargs.get("remote"):
+            patch = self.patches[kwargs["patch"]]
+            if not patch:
+                raise PatchError("No patch found at index %s in patchset." % kwargs["patch"])
+            (type, host, path, user, pswd, parm) = bb.decodeurl(patch["remote"])
+            if type == "file":
+                import shutil
+                if not patch.get("file") and patch.get("remote"):
+                    patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
+
+                shutil.copyfile(patch["quiltfile"], patch["file"])
+            else:
+                raise PatchError("Unable to do a remote refresh of %s, unsupported remote url scheme %s." % (os.path.basename(patch["quiltfile"]), type))
+        else:
+            # quilt refresh
+            args = ["refresh"]
+            if kwargs.get("quiltfile"):
+                args.append(os.path.basename(kwargs["quiltfile"]))
+            elif kwargs.get("patch"):
+                args.append(os.path.basename(self.patches[kwargs["patch"]]["quiltfile"]))
+            self._runcmd(args)
+
+class Resolver(object):
+    def __init__(self, patchset, terminal):
+        raise NotImplementedError()
+
+    def Resolve(self):
+        raise NotImplementedError()
+
+    def Revert(self):
+        raise NotImplementedError()
+
+    def Finalize(self):
+        raise NotImplementedError()
+
+class NOOPResolver(Resolver):
+    def __init__(self, patchset, terminal):
+        self.patchset = patchset
+        self.terminal = terminal
+
+    def Resolve(self):
+        olddir = os.path.abspath(os.curdir)
+        os.chdir(self.patchset.dir)
+        try:
+            self.patchset.Push()
+        except Exception:
+            import sys
+            os.chdir(olddir)
+            raise
+
+# Patch resolver which relies on the user doing all the work involved in the
+# resolution, with the exception of refreshing the remote copy of the patch
+# files (the urls).
+class UserResolver(Resolver):
+    def __init__(self, patchset, terminal):
+        self.patchset = patchset
+        self.terminal = terminal
+
+    # Force a push in the patchset, then drop to a shell for the user to
+    # resolve any rejected hunks
+    def Resolve(self):
+        olddir = os.path.abspath(os.curdir)
+        os.chdir(self.patchset.dir)
+        try:
+            self.patchset.Push(False)
+        except CmdError, v:
+            # Patch application failed
+            patchcmd = self.patchset.Push(True, False, False)
+
+            t = self.patchset.d.getVar('T', 1)
+            if not t:
+                bb.msg.fatal("Build", "T not set")
+            bb.utils.mkdirhier(t)
+            import random
+            rcfile = "%s/bashrc.%s.%s" % (t, str(os.getpid()), random.random())
+            f = open(rcfile, "w")
+            f.write("echo '*** Manual patch resolution mode ***'\n")
+            f.write("echo 'Dropping to a shell, so patch rejects can be fixed manually.'\n")
+            f.write("echo 'Run \"quilt refresh\" when patch is corrected, press CTRL+D to exit.'\n")
+            f.write("echo ''\n")
+            f.write(" ".join(patchcmd) + "\n")
+            f.close()
+            os.chmod(rcfile, 0775)
+
+            self.terminal("bash --rcfile " + rcfile, 'Patch Rejects: Please fix patch rejects manually', self.patchset.d)
+
+            # Construct a new PatchSet after the user's changes, compare the
+            # sets, checking patches for modifications, and doing a remote
+            # refresh on each.
+            oldpatchset = self.patchset
+            self.patchset = oldpatchset.__class__(self.patchset.dir, self.patchset.d)
+
+            for patch in self.patchset.patches:
+                oldpatch = None
+                for opatch in oldpatchset.patches:
+                    if opatch["quiltfile"] == patch["quiltfile"]:
+                        oldpatch = opatch
+
+                if oldpatch:
+                    patch["remote"] = oldpatch["remote"]
+                    if patch["quiltfile"] == oldpatch["quiltfile"]:
+                        if patch["quiltfilemd5"] != oldpatch["quiltfilemd5"]:
+                            bb.note("Patch %s has changed, updating remote url %s" % (os.path.basename(patch["quiltfile"]), patch["remote"]))
+                            # user change?  remote refresh
+                            self.patchset.Refresh(remote=True, patch=self.patchset.patches.index(patch))
+                        else:
+                            # User did not fix the problem.  Abort.
+                            raise PatchError("Patch application failed, and user did not fix and refresh the patch.")
+        except Exception:
+            os.chdir(olddir)
+            raise
+        os.chdir(olddir)

oe-core/meta/lib/oe/path.py

@@ -0,0 +1,130 @@
+import bb
+import errno
+import glob
+import os
+import shutil
+import subprocess
+
+def join(*paths):
+    """Like os.path.join but doesn't treat absolute RHS specially"""
+    return os.path.normpath("/".join(paths))
+
+def relative(src, dest):
+    """ Return a relative path from src to dest.
+
+    >>> relative("/usr/bin", "/tmp/foo/bar")
+    ../../tmp/foo/bar
+
+    >>> relative("/usr/bin", "/usr/lib")
+    ../lib
+
+    >>> relative("/tmp", "/tmp/foo/bar")
+    foo/bar
+    """
+
+    if hasattr(os.path, "relpath"):
+        return os.path.relpath(dest, src)
+    else:
+        destlist = os.path.normpath(dest).split(os.path.sep)
+        srclist = os.path.normpath(src).split(os.path.sep)
+
+        # Find common section of the path
+        common = os.path.commonprefix([destlist, srclist])
+        commonlen = len(common)
+
+        # Climb back to the point where they differentiate
+        relpath = [ os.path.pardir ] * (len(srclist) - commonlen)
+        if commonlen < len(destlist):
+            # Add remaining portion
+            relpath += destlist[commonlen:]
+
+        return os.path.sep.join(relpath)
+
+def format_display(path, metadata):
+    """ Prepare a path for display to the user. """
+    rel = relative(metadata.getVar("TOPDIR", 1), path)
+    if len(rel) > len(path):
+        return path
+    else:
+        return rel
+
+def copytree(src, dst):
+    # We could use something like shutil.copytree here but it turns out to
+    # to be slow. It takes twice as long copying to an empty directory. 
+    # If dst already has contents performance can be 15 time slower
+    # This way we also preserve hardlinks between files in the tree.
+
+    bb.utils.mkdirhier(dst)
+    cmd = 'tar -cf - -C %s -ps . | tar -xf - -C %s' % (src, dst)
+    check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+
+
+def remove(path, recurse=True):
+    """Equivalent to rm -f or rm -rf"""
+    for name in glob.glob(path):
+        try:
+            os.unlink(name)
+        except OSError, exc:
+            if recurse and exc.errno == errno.EISDIR:
+                shutil.rmtree(name)
+            elif exc.errno != errno.ENOENT:
+                raise
+
+def symlink(source, destination, force=False):
+    """Create a symbolic link"""
+    try:
+        if force:
+            remove(destination)
+        os.symlink(source, destination)
+    except OSError, e:
+        if e.errno != errno.EEXIST or os.readlink(destination) != source:
+            raise
+
+class CalledProcessError(Exception):
+    def __init__(self, retcode, cmd, output = None):
+        self.retcode = retcode
+        self.cmd = cmd
+        self.output = output
+    def __str__(self):
+        return "Command '%s' returned non-zero exit status %d with output %s" % (self.cmd, self.retcode, self.output)
+
+# Not needed when we move to python 2.7
+def check_output(*popenargs, **kwargs):
+    r"""Run command with arguments and return its output as a byte string.
+
+    If the exit code was non-zero it raises a CalledProcessError.  The
+    CalledProcessError object will have the return code in the returncode
+    attribute and output in the output attribute.
+
+    The arguments are the same as for the Popen constructor.  Example:
+
+    >>> check_output(["ls", "-l", "/dev/null"])
+    'crw-rw-rw- 1 root root 1, 3 Oct 18  2007 /dev/null\n'
+
+    The stdout argument is not allowed as it is used internally.
+    To capture standard error in the result, use stderr=STDOUT.
+
+    >>> check_output(["/bin/sh", "-c",
+    ...               "ls -l non_existent_file ; exit 0"],
+    ...              stderr=STDOUT)
+    'ls: non_existent_file: No such file or directory\n'
+    """
+    if 'stdout' in kwargs:
+        raise ValueError('stdout argument not allowed, it will be overridden.')
+    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+    output, unused_err = process.communicate()
+    retcode = process.poll()
+    if retcode:
+        cmd = kwargs.get("args")
+        if cmd is None:
+            cmd = popenargs[0]
+        raise CalledProcessError(retcode, cmd, output=output)
+    return output
+
+def find(dir, **walkoptions):
+    """ Given a directory, recurses into that directory,
+    returning all files as absolute paths. """
+
+    for root, dirs, files in os.walk(dir, **walkoptions):
+        for file in files:
+            yield os.path.join(root, file)

oe-core/meta/lib/oe/process.py

@@ -0,0 +1,74 @@
+import subprocess
+import signal
+
+def subprocess_setup():
+    # Python installs a SIGPIPE handler by default. This is usually not what
+    # non-Python subprocesses expect.
+    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+
+class CmdError(RuntimeError):
+    def __init__(self, command):
+        self.command = command
+
+    def __str__(self):
+        if not isinstance(self.command, basestring):
+            cmd = subprocess.list2cmdline(self.command)
+        else:
+            cmd = self.command
+
+        return "Execution of '%s' failed" % cmd
+
+class NotFoundError(CmdError):
+    def __str__(self):
+        return CmdError.__str__(self) + ": command not found"
+
+class ExecutionError(CmdError):
+    def __init__(self, command, exitcode, stdout = None, stderr = None):
+        CmdError.__init__(self, command)
+        self.exitcode = exitcode
+        self.stdout = stdout
+        self.stderr = stderr
+
+    def __str__(self):
+        message = ""
+        if self.stderr:
+            message += self.stderr
+        if self.stdout:
+            message += self.stdout
+        if message:
+            message = ":\n" + message
+        return (CmdError.__str__(self) +
+                " with exit code %s" % self.exitcode + message)
+
+class Popen(subprocess.Popen):
+    defaults = {
+        "close_fds": True,
+        "preexec_fn": subprocess_setup,
+        "stdout": subprocess.PIPE,
+        "stderr": subprocess.STDOUT,
+        "stdin": subprocess.PIPE,
+        "shell": False,
+    }
+
+    def __init__(self, *args, **kwargs):
+        options = dict(self.defaults)
+        options.update(kwargs)
+        subprocess.Popen.__init__(self, *args, **options)
+
+def run(cmd, input=None, **options):
+    """Convenience function to run a command and return its output, raising an
+    exception when the command fails"""
+
+    if isinstance(cmd, basestring) and not "shell" in options:
+        options["shell"] = True
+    try:
+        pipe = Popen(cmd, **options)
+    except OSError, exc:
+        if exc.errno == 2:
+            raise NotFoundError(cmd)
+        else:
+            raise
+    stdout, stderr = pipe.communicate(input)
+    if pipe.returncode != 0:
+        raise ExecutionError(cmd, pipe.returncode, stdout, stderr)
+    return stdout

oe-core/meta/lib/oe/prservice.py

@@ -0,0 +1,113 @@
+import bb
+
+def prserv_make_conn(d):
+    import prserv.serv
+    host = d.getVar("PRSERV_HOST",True)
+    port = d.getVar("PRSERV_PORT",True)
+    try:
+        conn = None
+        conn = prserv.serv.PRServerConnection(host,int(port))
+        d.setVar("__PRSERV_CONN",conn)
+    except Exception, exc:
+        bb.fatal("Connecting to PR service %s:%s failed: %s" % (host, port, str(exc)))
+
+    return conn
+
+def prserv_dump_db(d):
+    if d.getVar('USE_PR_SERV', True) != "1":
+        bb.error("Not using network based PR service")
+        return None
+
+    conn = d.getVar("__PRSERV_CONN", True)
+    if conn is None:
+        conn = prserv_make_conn(d)
+        if conn is None:
+            bb.error("Making connection failed to remote PR service")
+            return None
+
+    #dump db
+    opt_version = d.getVar('PRSERV_DUMPOPT_VERSION', True)
+    opt_pkgarch = d.getVar('PRSERV_DUMPOPT_PKGARCH', True)
+    opt_checksum = d.getVar('PRSERV_DUMPOPT_CHECKSUM', True)
+    opt_col = ("1" == d.getVar('PRSERV_DUMPOPT_COL', True))
+    return conn.export(opt_version, opt_pkgarch, opt_checksum, opt_col)
+
+def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksum=None):
+    if d.getVar('USE_PR_SERV', True) != "1":
+        bb.error("Not using network based PR service")
+        return None
+
+    conn = d.getVar("__PRSERV_CONN", True)
+    if conn is None:
+        conn = prserv_make_conn(d)
+        if conn is None:
+            bb.error("Making connection failed to remote PR service")
+            return None
+    #get the entry values
+    imported = []
+    prefix = "PRAUTO$"
+    for v in d.keys():
+        if v.startswith(prefix):
+            (remain, sep, checksum) = v.rpartition('$')
+            (remain, sep, pkgarch) = remain.rpartition('$')
+            (remain, sep, version) = remain.rpartition('$')
+            if (remain + '$' != prefix) or \
+               (filter_version and filter_version != version) or \
+               (filter_pkgarch and filter_pkgarch != pkgarch) or \
+               (filter_checksum and filter_checksum != checksum):
+               continue
+            try:
+                value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum, True))
+            except BaseException as exc:
+                bb.debug("Not valid value of %s:%s" % (v,str(exc)))
+                continue
+            ret = conn.importone(version,pkgarch,checksum,value)
+            if ret != value:
+                bb.error("importing(%s,%s,%s,%d) failed. DB may have larger value %d" % (version,pkgarch,checksum,value,ret))
+            else:
+               imported.append((version,pkgarch,checksum,value))
+    return imported
+
+def prserv_export_tofile(d, metainfo, datainfo, lockdown, nomax=False):
+    import bb.utils
+    #initilize the output file
+    bb.utils.mkdirhier(d.getVar('PRSERV_DUMPDIR', True))
+    df = d.getVar('PRSERV_DUMPFILE', True)
+    #write data
+    lf = bb.utils.lockfile("%s.lock" % df)
+    f = open(df, "a")
+    if metainfo:
+        #dump column info 
+        f.write("#PR_core_ver = \"%s\"\n\n" % metainfo['core_ver']);
+        f.write("#Table: %s\n" % metainfo['tbl_name'])
+        f.write("#Columns:\n")
+        f.write("#name      \t type    \t notn    \t dflt    \t pk\n")
+        f.write("#----------\t --------\t --------\t --------\t ----\n")
+        for i in range(len(metainfo['col_info'])):
+            f.write("#%10s\t %8s\t %8s\t %8s\t %4s\n" % 
+                    (metainfo['col_info'][i]['name'], 
+                     metainfo['col_info'][i]['type'], 
+                     metainfo['col_info'][i]['notnull'], 
+                     metainfo['col_info'][i]['dflt_value'], 
+                     metainfo['col_info'][i]['pk']))
+        f.write("\n")
+
+    if lockdown:
+        f.write("PRSERV_LOCKDOWN = \"1\"\n\n")
+
+    if datainfo:
+        idx = {}
+        for i in range(len(datainfo)):
+            pkgarch = datainfo[i]['pkgarch']
+            value = datainfo[i]['value']
+            if not idx.has_key(pkgarch):
+                idx[pkgarch] = i
+            elif value > datainfo[idx[pkgarch]]['value']:
+                idx[pkgarch] = i
+            f.write("PRAUTO$%s$%s$%s = \"%s\"\n" % 
+                (str(datainfo[i]['version']), pkgarch, str(datainfo[i]['checksum']), str(value)))
+        if not nomax:
+            for i in idx:
+                f.write("PRAUTO_%s_%s = \"%s\"\n" % (str(datainfo[idx[i]]['version']),str(datainfo[idx[i]]['pkgarch']),str(datainfo[idx[i]]['value'])))
+    f.close()
+    bb.utils.unlockfile(lf)

oe-core/meta/lib/oe/qa.py

@@ -0,0 +1,89 @@
+class ELFFile:
+    EI_NIDENT = 16
+
+    EI_CLASS      = 4
+    EI_DATA       = 5
+    EI_VERSION    = 6
+    EI_OSABI      = 7
+    EI_ABIVERSION = 8
+
+    # possible values for EI_CLASS
+    ELFCLASSNONE = 0
+    ELFCLASS32   = 1
+    ELFCLASS64   = 2
+
+    # possible value for EI_VERSION
+    EV_CURRENT   = 1
+
+    # possible values for EI_DATA
+    ELFDATANONE  = 0
+    ELFDATA2LSB  = 1
+    ELFDATA2MSB  = 2
+
+    def my_assert(self, expectation, result):
+        if not expectation == result:
+            #print "'%x','%x' %s" % (ord(expectation), ord(result), self.name)
+            raise Exception("This does not work as expected")
+
+    def __init__(self, name, bits = 0):
+        self.name = name
+        self.bits = bits
+
+    def open(self):
+        self.file = file(self.name, "r")
+        self.data = self.file.read(ELFFile.EI_NIDENT+4)
+
+        self.my_assert(len(self.data), ELFFile.EI_NIDENT+4)
+        self.my_assert(self.data[0], chr(0x7f) )
+        self.my_assert(self.data[1], 'E')
+        self.my_assert(self.data[2], 'L')
+        self.my_assert(self.data[3], 'F')
+        if self.bits == 0:
+            if self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS32):
+                self.bits = 32
+            elif self.data[ELFFile.EI_CLASS] == chr(ELFFile.ELFCLASS64):
+                self.bits = 64
+            else:
+                # Not 32-bit or 64.. lets assert
+                raise Exception("ELF but not 32 or 64 bit.")
+        elif self.bits == 32:
+            self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS32))
+        elif self.bits == 64:
+            self.my_assert(self.data[ELFFile.EI_CLASS], chr(ELFFile.ELFCLASS64))
+        else:
+            raise Exception("Must specify unknown, 32 or 64 bit size.")
+        self.my_assert(self.data[ELFFile.EI_VERSION], chr(ELFFile.EV_CURRENT) )
+
+        self.sex = self.data[ELFFile.EI_DATA]
+        if self.sex == chr(ELFFile.ELFDATANONE):
+            raise Exception("self.sex == ELFDATANONE")
+        elif self.sex == chr(ELFFile.ELFDATA2LSB):
+            self.sex = "<"
+        elif self.sex == chr(ELFFile.ELFDATA2MSB):
+            self.sex = ">"
+        else:
+            raise Exception("Unknown self.sex")
+
+    def osAbi(self):
+        return ord(self.data[ELFFile.EI_OSABI])
+
+    def abiVersion(self):
+        return ord(self.data[ELFFile.EI_ABIVERSION])
+
+    def abiSize(self):
+	return self.bits
+
+    def isLittleEndian(self):
+        return self.sex == "<"
+
+    def isBigEngian(self):
+        return self.sex == ">"
+
+    def machine(self):
+        """
+        We know the sex stored in self.sex and we
+        know the position
+        """
+        import struct
+        (a,) = struct.unpack(self.sex+"H", self.data[18:20])
+        return a

oe-core/meta/lib/oe/sstatesig.py

@@ -0,0 +1,52 @@
+import bb.siggen
+
+def sstate_rundepfilter(fn, recipename, task, dep, depname):
+    # Return True if we should keep the dependency, False to drop it
+    def isNative(x):
+        return x.endswith("-native")
+    def isCross(x):
+        return x.endswith("-cross") or x.endswith("-cross-initial") or x.endswith("-cross-intermediate")
+    def isNativeSDK(x):
+        return x.endswith("-nativesdk")
+
+    # Always include our own inter-task dependencies
+    if recipename == depname:
+        return True
+
+    # Quilt (patch application) changing isn't likely to affect anything
+    if depname == "quilt-native":
+        return False
+    # Don't change native/cross/nativesdk recipe dependencies any further
+    if isNative(recipename) or isCross(recipename) or isNativeSDK(recipename):
+        return True
+
+    # Only target packages beyond here
+
+    # Drop native/cross/nativesdk dependencies from target recipes
+    if isNative(depname) or isCross(depname) or isNativeSDK(depname):
+        return False
+
+    # Exclude well defined machine specific configurations which don't change ABI
+    if depname in ['sysvinit-inittab', 'shadow-securetty', 'opkg-config-base', 'netbase', 'formfactor', 'xserver-xf86-config', 'pointercal', 'base-files']:
+        return False
+
+    # Default to keep dependencies
+    return True
+
+class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
+    name = "OEBasic"
+    def init_rundepcheck(self, data):
+        pass
+    def rundep_check(self, fn, recipename, task, dep, depname):
+        return sstate_rundepfilter(fn, recipename, task, dep, depname)
+
+class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
+    name = "OEBasicHash"
+    def init_rundepcheck(self, data):
+        pass
+    def rundep_check(self, fn, recipename, task, dep, depname):
+        return sstate_rundepfilter(fn, recipename, task, dep, depname)
+
+# Insert these classes into siggen's namespace so it can see and select them
+bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic
+bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash

oe-core/meta/lib/oe/terminal.py

@@ -0,0 +1,156 @@
+import logging
+import os
+import oe.classutils
+import shlex
+from bb.process import Popen, ExecutionError
+
+logger = logging.getLogger('BitBake.OE.Terminal')
+
+
+class UnsupportedTerminal(StandardError):
+    pass
+
+class NoSupportedTerminals(StandardError):
+    pass
+
+
+class Registry(oe.classutils.ClassRegistry):
+    command = None
+
+    def __init__(cls, name, bases, attrs):
+        super(Registry, cls).__init__(name.lower(), bases, attrs)
+
+    @property
+    def implemented(cls):
+        return bool(cls.command)
+
+
+class Terminal(Popen):
+    __metaclass__ = Registry
+
+    def __init__(self, command, title=None, env=None):
+        self.format_command(command, title)
+
+        try:
+            Popen.__init__(self, self.command, env=env)
+        except OSError as exc:
+            import errno
+            if exc.errno == errno.ENOENT:
+                raise UnsupportedTerminal(self.name)
+            else:
+                raise
+
+    def format_command(self, command, title):
+        fmt = {'title': title or 'Terminal', 'command': command}
+        if isinstance(self.command, basestring):
+            self.command = shlex.split(self.command.format(**fmt))
+        else:
+            self.command = [element.format(**fmt) for element in self.command]
+
+class XTerminal(Terminal):
+    def __init__(self, command, title=None, env=None):
+        Terminal.__init__(self, command, title, env)
+        if not os.environ.get('DISPLAY'):
+            raise UnsupportedTerminal(self.name)
+
+class Gnome(XTerminal):
+    command = 'gnome-terminal --disable-factory -t "{title}" -x {command}'
+    priority = 2
+
+class Xfce(XTerminal):
+    command = 'Terminal -T "{title}" -e "{command}"'
+    priority = 2
+
+    def __init__(self, command, title=None, env=None):
+        # Upstream binary name is Terminal but Debian/Ubuntu use
+        # xfce4-terminal to avoid possible(?) conflicts
+        distro = distro_name()
+        if distro == 'ubuntu' or distro == 'debian':
+            cmd = 'xfce4-terminal -T "{title}" -e "{command}"'
+        else:
+            cmd = command
+        XTerminal.__init__(self, cmd, title, env)
+
+class Konsole(XTerminal):
+    command = 'konsole -T "{title}" -e {command}'
+    priority = 2
+
+    def __init__(self, command, title=None, env=None):
+        # Check version
+        vernum = check_konsole_version("konsole")
+        if vernum:
+            if vernum.split('.')[0] == "2":
+                logger.debug(1, 'Konsole from KDE 4.x will not work as devshell, skipping')
+                raise UnsupportedTerminal(self.name)
+        XTerminal.__init__(self, command, title, env)
+
+class XTerm(XTerminal):
+    command = 'xterm -T "{title}" -e {command}'
+    priority = 1
+
+class Rxvt(XTerminal):
+    command = 'rxvt -T "{title}" -e {command}'
+    priority = 1
+
+class Screen(Terminal):
+    command = 'screen -D -m -t "{title}" -S devshell {command}'
+
+    def __init__(self, command, title=None, env=None):
+        Terminal.__init__(self, command, title, env)
+        logger.warn('Screen started. Please connect in another terminal with '
+                    '"screen -r devshell"')
+
+
+def prioritized():
+    return Registry.prioritized()
+
+def spawn_preferred(command, title=None, env=None):
+    """Spawn the first supported terminal, by priority"""
+    for terminal in prioritized():
+        try:
+            spawn(terminal.name, command, title, env)
+            break
+        except UnsupportedTerminal:
+            continue
+    else:
+        raise NoSupportedTerminals()
+
+def spawn(name, command, title=None, env=None):
+    """Spawn the specified terminal, by name"""
+    logger.debug(1, 'Attempting to spawn terminal "%s"', name)
+    try:
+        terminal = Registry.registry[name]
+    except KeyError:
+        raise UnsupportedTerminal(name)
+
+    pipe = terminal(command, title, env)
+    output = pipe.communicate()[0]
+    if pipe.returncode != 0:
+        raise ExecutionError(pipe.command, pipe.returncode, output)
+
+def check_konsole_version(konsole):
+    import subprocess as sub
+    try:
+        p = sub.Popen(['sh', '-c', '%s --version' % konsole],stdout=sub.PIPE,stderr=sub.PIPE)
+        out, err = p.communicate()
+        ver_info = out.rstrip().split('\n')
+    except OSError as exc:
+        import errno
+        if exc.errno == errno.ENOENT:
+            return None
+        else:
+            raise
+    vernum = None
+    for ver in ver_info:
+        if ver.startswith('Konsole'):
+            vernum = ver.split(' ')[-1]
+    return vernum
+
+def distro_name():
+    try:
+        p = Popen(['lsb_release', '-i'])
+        out, err = p.communicate()
+        distro = out.split(':')[1].strip().lower()
+    except:
+        distro = "unknown"
+    return distro

oe-core/meta/lib/oe/tests/test_license.py

@@ -0,0 +1,68 @@
+import unittest
+import oe.license
+
+class SeenVisitor(oe.license.LicenseVisitor):
+    def __init__(self):
+        self.seen = []
+        oe.license.LicenseVisitor.__init__(self)
+
+    def visit_Str(self, node):
+        self.seen.append(node.s)
+
+class TestSingleLicense(unittest.TestCase):
+    licenses = [
+        "GPLv2",
+        "LGPL-2.0",
+        "Artistic",
+        "MIT",
+        "GPLv3+",
+        "FOO_BAR",
+    ]
+    invalid_licenses = ["GPL/BSD"]
+
+    @staticmethod
+    def parse(licensestr):
+        visitor = SeenVisitor()
+        visitor.visit_string(licensestr)
+        return visitor.seen
+
+    def test_single_licenses(self):
+        for license in self.licenses:
+            licenses = self.parse(license)
+            self.assertListEqual(licenses, [license])
+
+    def test_invalid_licenses(self):
+        for license in self.invalid_licenses:
+            with self.assertRaises(oe.license.InvalidLicense) as cm:
+                self.parse(license)
+            self.assertEqual(cm.exception.license, license)
+
+class TestSimpleCombinations(unittest.TestCase):
+    tests = {
+        "FOO&BAR": ["FOO", "BAR"],
+        "BAZ & MOO": ["BAZ", "MOO"],
+        "ALPHA|BETA": ["ALPHA"],
+        "BAZ&MOO|FOO": ["FOO"],
+        "FOO&BAR|BAZ": ["FOO", "BAR"],
+    }
+    preferred = ["ALPHA", "FOO", "BAR"]
+
+    def test_tests(self):
+        def choose(a, b):
+            if all(lic in self.preferred for lic in b):
+                return b
+            else:
+                return a
+
+        for license, expected in self.tests.items():
+            licenses = oe.license.flattened_licenses(license, choose)
+            self.assertListEqual(licenses, expected)
+
+class TestComplexCombinations(TestSimpleCombinations):
+    tests = {
+        "FOO & (BAR | BAZ)&MOO": ["FOO", "BAR", "MOO"],
+        "(ALPHA|(BETA&THETA)|OMEGA)&DELTA": ["OMEGA", "DELTA"],
+        "((ALPHA|BETA)&FOO)|BAZ": ["BETA", "FOO"],
+        "(GPL-2.0|Proprietary)&BSD-4-clause&MIT": ["GPL-2.0", "BSD-4-clause", "MIT"],
+    }
+    preferred = ["BAR", "OMEGA", "BETA", "GPL-2.0"]

oe-core/meta/lib/oe/tests/test_types.py

@@ -0,0 +1,62 @@
+import unittest
+from oe.maketype import create, factory
+
+class TestTypes(unittest.TestCase):
+    def assertIsInstance(self, obj, cls):
+        return self.assertTrue(isinstance(obj, cls))
+
+    def assertIsNot(self, obj, other):
+        return self.assertFalse(obj is other)
+
+    def assertFactoryCreated(self, value, type, **flags):
+        cls = factory(type)
+        self.assertIsNot(cls, None)
+        self.assertIsInstance(create(value, type, **flags), cls)
+
+class TestBooleanType(TestTypes):
+    def test_invalid(self):
+        self.assertRaises(ValueError, create, '', 'boolean')
+        self.assertRaises(ValueError, create, 'foo', 'boolean')
+        self.assertRaises(TypeError, create, object(), 'boolean')
+
+    def test_true(self):
+        self.assertTrue(create('y', 'boolean'))
+        self.assertTrue(create('yes', 'boolean'))
+        self.assertTrue(create('1', 'boolean'))
+        self.assertTrue(create('t', 'boolean'))
+        self.assertTrue(create('true', 'boolean'))
+        self.assertTrue(create('TRUE', 'boolean'))
+        self.assertTrue(create('truE', 'boolean'))
+
+    def test_false(self):
+        self.assertFalse(create('n', 'boolean'))
+        self.assertFalse(create('no', 'boolean'))
+        self.assertFalse(create('0', 'boolean'))
+        self.assertFalse(create('f', 'boolean'))
+        self.assertFalse(create('false', 'boolean'))
+        self.assertFalse(create('FALSE', 'boolean'))
+        self.assertFalse(create('faLse', 'boolean'))
+
+    def test_bool_equality(self):
+        self.assertEqual(create('n', 'boolean'), False)
+        self.assertNotEqual(create('n', 'boolean'), True)
+        self.assertEqual(create('y', 'boolean'), True)
+        self.assertNotEqual(create('y', 'boolean'), False)
+
+class TestList(TestTypes):
+    def assertListEqual(self, value, valid, sep=None):
+        obj = create(value, 'list', separator=sep)
+        self.assertEqual(obj, valid)
+        if sep is not None:
+            self.assertEqual(obj.separator, sep)
+        self.assertEqual(str(obj), obj.separator.join(obj))
+
+    def test_list_nosep(self):
+        testlist = ['alpha', 'beta', 'theta']
+        self.assertListEqual('alpha beta theta', testlist)
+        self.assertListEqual('alpha  beta\ttheta', testlist)
+        self.assertListEqual('alpha', ['alpha'])
+
+    def test_list_usersep(self):
+        self.assertListEqual('foo:bar', ['foo', 'bar'], ':')
+        self.assertListEqual('foo:bar:baz', ['foo', 'bar', 'baz'], ':')

oe-core/meta/lib/oe/types.py

@@ -0,0 +1,104 @@
+import re
+
+class OEList(list):
+    """OpenEmbedded 'list' type
+
+    Acts as an ordinary list, but is constructed from a string value and a
+    separator (optional), and re-joins itself when converted to a string with
+    str().  Set the variable type flag to 'list' to use this type, and the
+    'separator' flag may be specified (defaulting to whitespace)."""
+
+    name = "list"
+
+    def __init__(self, value, separator = None):
+        if value is not None:
+            list.__init__(self, value.split(separator))
+        else:
+            list.__init__(self)
+
+        if separator is None:
+            self.separator = " "
+        else:
+            self.separator = separator
+
+    def __str__(self):
+        return self.separator.join(self)
+
+def choice(value, choices):
+    """OpenEmbedded 'choice' type
+
+    Acts as a multiple choice for the user.  To use this, set the variable
+    type flag to 'choice', and set the 'choices' flag to a space separated
+    list of valid values."""
+    if not isinstance(value, basestring):
+        raise TypeError("choice accepts a string, not '%s'" % type(value))
+
+    value = value.lower()
+    choices = choices.lower()
+    if value not in choices.split():
+        raise ValueError("Invalid choice '%s'.  Valid choices: %s" %
+                         (value, choices))
+    return value
+
+def regex(value, regexflags=None):
+    """OpenEmbedded 'regex' type
+
+    Acts as a regular expression, returning the pre-compiled regular
+    expression pattern object.  To use this type, set the variable type flag
+    to 'regex', and optionally, set the 'regexflags' type to a space separated
+    list of the flags to control the regular expression matching (e.g.
+    FOO[regexflags] += 'ignorecase').  See the python documentation on the
+    're' module for a list of valid flags."""
+
+    flagval = 0
+    if regexflags:
+        for flag in regexflags.split():
+            flag = flag.upper()
+            try:
+                flagval |= getattr(re, flag)
+            except AttributeError:
+                raise ValueError("Invalid regex flag '%s'" % flag)
+
+    try:
+        return re.compile(value, flagval)
+    except re.error, exc:
+        raise ValueError("Invalid regex value '%s': %s" %
+                         (value, exc.args[0]))
+
+def boolean(value):
+    """OpenEmbedded 'boolean' type
+
+    Valid values for true: 'yes', 'y', 'true', 't', '1'
+    Valid values for false: 'no', 'n', 'false', 'f', '0'
+    """
+
+    if not isinstance(value, basestring):
+        raise TypeError("boolean accepts a string, not '%s'" % type(value))
+
+    value = value.lower()
+    if value in ('yes', 'y', 'true', 't', '1'):
+        return True
+    elif value in ('no', 'n', 'false', 'f', '0'):
+        return False
+    raise ValueError("Invalid boolean value '%s'" % value)
+
+def integer(value, numberbase=10):
+    """OpenEmbedded 'integer' type
+
+    Defaults to base 10, but this can be specified using the optional
+    'numberbase' flag."""
+
+    return int(value, int(numberbase))
+
+_float = float
+def float(value, fromhex='false'):
+    """OpenEmbedded floating point type
+
+    To use this type, set the type flag to 'float', and optionally set the
+    'fromhex' flag to a true value (obeying the same rules as for the
+    'boolean' type) if the value is in base 16 rather than base 10."""
+
+    if boolean(fromhex):
+        return _float.fromhex(value)
+    else:
+        return _float(value)

oe-core/meta/lib/oe/utils.py

@@ -0,0 +1,90 @@
+import bb, bb.data
+
+def read_file(filename):
+    try:
+        f = file( filename, "r" )
+    except IOError, reason:
+        return "" # WARNING: can't raise an error now because of the new RDEPENDS handling. This is a bit ugly. :M:
+    else:
+        return f.read().strip()
+    return None
+
+def ifelse(condition, iftrue = True, iffalse = False):
+    if condition:
+        return iftrue
+    else:
+        return iffalse
+
+def conditional(variable, checkvalue, truevalue, falsevalue, d):
+    if d.getVar(variable,1) == checkvalue:
+        return truevalue
+    else:
+        return falsevalue
+
+def less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
+    if float(d.getVar(variable,1)) <= float(checkvalue):
+        return truevalue
+    else:
+        return falsevalue
+
+def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
+    result = bb.vercmp(d.getVar(variable,True), checkvalue)
+    if result <= 0:
+        return truevalue
+    else:
+        return falsevalue
+
+def contains(variable, checkvalues, truevalue, falsevalue, d):
+    val = d.getVar(variable, True)
+    if not val:
+        return falsevalue
+    val = set(val.split())
+    if isinstance(checkvalues, basestring):
+        checkvalues = set(checkvalues.split())
+    else:
+        checkvalues = set(checkvalues)
+    if checkvalues.issubset(val):
+        return truevalue
+    return falsevalue
+
+def both_contain(variable1, variable2, checkvalue, d):
+    if d.getVar(variable1,1).find(checkvalue) != -1 and d.getVar(variable2,1).find(checkvalue) != -1:
+        return checkvalue
+    else:
+        return ""
+
+def prune_suffix(var, suffixes, d):
+    # See if var ends with any of the suffixes listed and 
+    # remove it if found
+    for suffix in suffixes:
+        if var.endswith(suffix):
+            var = var.replace(suffix, "")
+
+    prefix = d.getVar("MLPREFIX", True)
+    if prefix and var.startswith(prefix):
+        var = var.replace(prefix, "")
+
+    return var
+
+def str_filter(f, str, d):
+    from re import match
+    return " ".join(filter(lambda x: match(f, x, 0), str.split()))
+
+def str_filter_out(f, str, d):
+    from re import match
+    return " ".join(filter(lambda x: not match(f, x, 0), str.split()))
+
+def param_bool(cfg, field, dflt = None):
+    """Lookup <field> in <cfg> map and convert it to a boolean; take
+    <dflt> when this <field> does not exist"""
+    value = cfg.get(field, dflt)
+    strvalue = str(value).lower()
+    if strvalue in ('yes', 'y', 'true', 't', '1'):
+        return True
+    elif strvalue in ('no', 'n', 'false', 'f', '0'):
+        return False
+    raise ValueError("invalid value for boolean parameter '%s': '%s'" % (field, value))
+
+def inherits(d, *classes):
+    """Return True if the metadata inherits any of the specified classes"""
+    return any(bb.data.inherits_class(cls, d) for cls in classes)

oe-core/meta/lib/test.py

@@ -0,0 +1,3 @@
+import oe.license
+
+print(oe.license.is_included('LGPLv2.1 | GPLv3', ['*'], []))