418 lines
16 KiB
Python
418 lines
16 KiB
Python
|
# Recipe creation tool - create command plugin
|
||
|
|
#
|
||
|
|
# Copyright (C) 2014 Intel Corporation
|
||
|
|
#
|
||
|
|
# This program is free software; you can redistribute it and/or modify
|
||
|
|
# it under the terms of the GNU General Public License version 2 as
|
||
|
|
# published by the Free Software Foundation.
|
||
|
|
#
|
||
|
|
# This program is distributed in the hope that it will be useful,
|
||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
# GNU General Public License for more details.
|
||
|
|
#
|
||
|
|
# You should have received a copy of the GNU General Public License along
|
||
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||
|
|
|
||
|
|
import sys
|
||
|
|
import os
|
||
|
|
import argparse
|
||
|
|
import glob
|
||
|
|
import fnmatch
|
||
|
|
import re
|
||
|
|
import logging
|
||
|
|
|
||
|
|
logger = logging.getLogger('recipetool')
|
||
|
|
|
||
|
|
tinfoil = None
|
||
|
|
plugins = None
|
||
|
|
|
||
|
|
def plugin_init(pluginlist):
|
||
|
|
# Take a reference to the list so we can use it later
|
||
|
|
global plugins
|
||
|
|
plugins = pluginlist
|
||
|
|
|
||
|
|
def tinfoil_init(instance):
|
||
|
|
global tinfoil
|
||
|
|
tinfoil = instance
|
||
|
|
|
||
|
|
class RecipeHandler():
|
||
|
|
@staticmethod
|
||
|
|
def checkfiles(path, speclist):
|
||
|
|
results = []
|
||
|
|
for spec in speclist:
|
||
|
|
results.extend(glob.glob(os.path.join(path, spec)))
|
||
|
|
return results
|
||
|
|
|
||
|
|
def genfunction(self, outlines, funcname, content):
|
||
|
|
outlines.append('%s () {' % funcname)
|
||
|
|
for line in content:
|
||
|
|
outlines.append('\t%s' % line)
|
||
|
|
outlines.append('}')
|
||
|
|
outlines.append('')
|
||
|
|
|
||
|
|
def process(self, srctree, classes, lines_before, lines_after, handled):
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def fetch_source(uri, destdir):
|
||
|
|
import bb.data
|
||
|
|
bb.utils.mkdirhier(destdir)
|
||
|
|
localdata = bb.data.createCopy(tinfoil.config_data)
|
||
|
|
bb.data.update_data(localdata)
|
||
|
|
localdata.setVar('BB_STRICT_CHECKSUM', '')
|
||
|
|
localdata.setVar('SRCREV', '${AUTOREV}')
|
||
|
|
ret = (None, None)
|
||
|
|
olddir = os.getcwd()
|
||
|
|
try:
|
||
|
|
fetcher = bb.fetch2.Fetch([uri], localdata)
|
||
|
|
for u in fetcher.ud:
|
||
|
|
ud = fetcher.ud[u]
|
||
|
|
ud.ignore_checksums = True
|
||
|
|
fetcher.download()
|
||
|
|
fetcher.unpack(destdir)
|
||
|
|
for u in fetcher.ud:
|
||
|
|
ud = fetcher.ud[u]
|
||
|
|
if ud.method.recommends_checksum(ud):
|
||
|
|
md5value = bb.utils.md5_file(ud.localpath)
|
||
|
|
sha256value = bb.utils.sha256_file(ud.localpath)
|
||
|
|
ret = (md5value, sha256value)
|
||
|
|
except bb.fetch2.BBFetchException, e:
|
||
|
|
raise bb.build.FuncFailed(e)
|
||
|
|
finally:
|
||
|
|
os.chdir(olddir)
|
||
|
|
return ret
|
||
|
|
|
||
|
|
def supports_srcrev(uri):
|
||
|
|
localdata = bb.data.createCopy(tinfoil.config_data)
|
||
|
|
bb.data.update_data(localdata)
|
||
|
|
fetcher = bb.fetch2.Fetch([uri], localdata)
|
||
|
|
urldata = fetcher.ud
|
||
|
|
for u in urldata:
|
||
|
|
if urldata[u].method.supports_srcrev():
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def create_recipe(args):
|
||
|
|
import bb.process
|
||
|
|
import tempfile
|
||
|
|
import shutil
|
||
|
|
|
||
|
|
pkgarch = ""
|
||
|
|
if args.machine:
|
||
|
|
pkgarch = "${MACHINE_ARCH}"
|
||
|
|
|
||
|
|
checksums = (None, None)
|
||
|
|
tempsrc = ''
|
||
|
|
srcsubdir = ''
|
||
|
|
if '://' in args.source:
|
||
|
|
# Fetch a URL
|
||
|
|
srcuri = args.source
|
||
|
|
if args.extract_to:
|
||
|
|
srctree = args.extract_to
|
||
|
|
else:
|
||
|
|
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
|
||
|
|
srctree = tempsrc
|
||
|
|
logger.info('Fetching %s...' % srcuri)
|
||
|
|
checksums = fetch_source(args.source, srctree)
|
||
|
|
dirlist = os.listdir(srctree)
|
||
|
|
if 'git.indirectionsymlink' in dirlist:
|
||
|
|
dirlist.remove('git.indirectionsymlink')
|
||
|
|
if len(dirlist) == 1 and os.path.isdir(os.path.join(srctree, dirlist[0])):
|
||
|
|
# We unpacked a single directory, so we should use that
|
||
|
|
srcsubdir = dirlist[0]
|
||
|
|
srctree = os.path.join(srctree, srcsubdir)
|
||
|
|
else:
|
||
|
|
# Assume we're pointing to an existing source tree
|
||
|
|
if args.extract_to:
|
||
|
|
logger.error('--extract-to cannot be specified if source is a directory')
|
||
|
|
sys.exit(1)
|
||
|
|
if not os.path.isdir(args.source):
|
||
|
|
logger.error('Invalid source directory %s' % args.source)
|
||
|
|
sys.exit(1)
|
||
|
|
srcuri = ''
|
||
|
|
srctree = args.source
|
||
|
|
|
||
|
|
outfile = args.outfile
|
||
|
|
if outfile and outfile != '-':
|
||
|
|
if os.path.exists(outfile):
|
||
|
|
logger.error('Output file %s already exists' % outfile)
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
lines_before = []
|
||
|
|
lines_after = []
|
||
|
|
|
||
|
|
lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0]))
|
||
|
|
lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.')
|
||
|
|
lines_before.append('# (Feel free to remove these comments when editing.)')
|
||
|
|
lines_before.append('#')
|
||
|
|
|
||
|
|
licvalues = guess_license(srctree)
|
||
|
|
lic_files_chksum = []
|
||
|
|
if licvalues:
|
||
|
|
licenses = []
|
||
|
|
for licvalue in licvalues:
|
||
|
|
if not licvalue[0] in licenses:
|
||
|
|
licenses.append(licvalue[0])
|
||
|
|
lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
|
||
|
|
lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
|
||
|
|
lines_before.append('# your responsibility to verify that the values are complete and correct.')
|
||
|
|
if len(licvalues) > 1:
|
||
|
|
lines_before.append('#')
|
||
|
|
lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate')
|
||
|
|
lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there')
|
||
|
|
lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying')
|
||
|
|
lines_before.append('# documentation to determine which situation is applicable.')
|
||
|
|
else:
|
||
|
|
lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
|
||
|
|
lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
|
||
|
|
lines_before.append('#')
|
||
|
|
lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
|
||
|
|
lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
|
||
|
|
lines_before.append('# will not be in most cases) you must specify the correct value before using this')
|
||
|
|
lines_before.append('# recipe for anything other than initial testing/development!')
|
||
|
|
licenses = ['CLOSED']
|
||
|
|
lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
|
||
|
|
lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
|
||
|
|
lines_before.append('')
|
||
|
|
|
||
|
|
# FIXME This is kind of a hack, we probably ought to be using bitbake to do this
|
||
|
|
# we'd also want a way to automatically set outfile based upon auto-detecting these values from the source if possible
|
||
|
|
recipefn = os.path.splitext(os.path.basename(outfile))[0]
|
||
|
|
fnsplit = recipefn.split('_')
|
||
|
|
if len(fnsplit) > 1:
|
||
|
|
pn = fnsplit[0]
|
||
|
|
pv = fnsplit[1]
|
||
|
|
else:
|
||
|
|
pn = recipefn
|
||
|
|
pv = None
|
||
|
|
|
||
|
|
if srcuri:
|
||
|
|
if pv and pv not in 'git svn hg'.split():
|
||
|
|
srcuri = srcuri.replace(pv, '${PV}')
|
||
|
|
else:
|
||
|
|
lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
|
||
|
|
lines_before.append('SRC_URI = "%s"' % srcuri)
|
||
|
|
(md5value, sha256value) = checksums
|
||
|
|
if md5value:
|
||
|
|
lines_before.append('SRC_URI[md5sum] = "%s"' % md5value)
|
||
|
|
if sha256value:
|
||
|
|
lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value)
|
||
|
|
if srcuri and supports_srcrev(srcuri):
|
||
|
|
lines_before.append('')
|
||
|
|
lines_before.append('# Modify these as desired')
|
||
|
|
lines_before.append('PV = "1.0+git${SRCPV}"')
|
||
|
|
lines_before.append('SRCREV = "${AUTOREV}"')
|
||
|
|
lines_before.append('')
|
||
|
|
|
||
|
|
if srcsubdir and pv:
|
||
|
|
if srcsubdir == "%s-%s" % (pn, pv):
|
||
|
|
# This would be the default, so we don't need to set S in the recipe
|
||
|
|
srcsubdir = ''
|
||
|
|
if srcsubdir:
|
||
|
|
if pv and pv not in 'git svn hg'.split():
|
||
|
|
srcsubdir = srcsubdir.replace(pv, '${PV}')
|
||
|
|
lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
|
||
|
|
lines_before.append('')
|
||
|
|
|
||
|
|
if pkgarch:
|
||
|
|
lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch)
|
||
|
|
lines_after.append('')
|
||
|
|
|
||
|
|
# Find all plugins that want to register handlers
|
||
|
|
handlers = []
|
||
|
|
for plugin in plugins:
|
||
|
|
if hasattr(plugin, 'register_recipe_handlers'):
|
||
|
|
plugin.register_recipe_handlers(handlers)
|
||
|
|
|
||
|
|
# Apply the handlers
|
||
|
|
classes = []
|
||
|
|
handled = []
|
||
|
|
for handler in handlers:
|
||
|
|
handler.process(srctree, classes, lines_before, lines_after, handled)
|
||
|
|
|
||
|
|
outlines = []
|
||
|
|
outlines.extend(lines_before)
|
||
|
|
if classes:
|
||
|
|
outlines.append('inherit %s' % ' '.join(classes))
|
||
|
|
outlines.append('')
|
||
|
|
outlines.extend(lines_after)
|
||
|
|
|
||
|
|
if outfile == '-':
|
||
|
|
sys.stdout.write('\n'.join(outlines) + '\n')
|
||
|
|
else:
|
||
|
|
with open(outfile, 'w') as f:
|
||
|
|
f.write('\n'.join(outlines) + '\n')
|
||
|
|
logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile)
|
||
|
|
|
||
|
|
if tempsrc:
|
||
|
|
shutil.rmtree(tempsrc)
|
||
|
|
|
||
|
|
return 0
|
||
|
|
|
||
|
|
def get_license_md5sums(d, static_only=False):
|
||
|
|
import bb.utils
|
||
|
|
md5sums = {}
|
||
|
|
if not static_only:
|
||
|
|
# Gather md5sums of license files in common license dir
|
||
|
|
commonlicdir = d.getVar('COMMON_LICENSE_DIR', True)
|
||
|
|
for fn in os.listdir(commonlicdir):
|
||
|
|
md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
|
||
|
|
md5sums[md5value] = fn
|
||
|
|
# The following were extracted from common values in various recipes
|
||
|
|
# (double checking the license against the license file itself, not just
|
||
|
|
# the LICENSE value in the recipe)
|
||
|
|
md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2'
|
||
|
|
md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2'
|
||
|
|
md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2'
|
||
|
|
md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2'
|
||
|
|
md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2'
|
||
|
|
md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2'
|
||
|
|
md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2'
|
||
|
|
md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2'
|
||
|
|
md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2'
|
||
|
|
md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2'
|
||
|
|
md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2'
|
||
|
|
md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2'
|
||
|
|
md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2'
|
||
|
|
md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2'
|
||
|
|
md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file
|
||
|
|
md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1'
|
||
|
|
md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1'
|
||
|
|
md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1'
|
||
|
|
md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1'
|
||
|
|
md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1'
|
||
|
|
md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1'
|
||
|
|
md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1'
|
||
|
|
md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1'
|
||
|
|
md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2'
|
||
|
|
md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2'
|
||
|
|
md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2'
|
||
|
|
md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2'
|
||
|
|
md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2'
|
||
|
|
md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3'
|
||
|
|
md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3'
|
||
|
|
md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3'
|
||
|
|
md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0'
|
||
|
|
md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules
|
||
|
|
return md5sums
|
||
|
|
|
||
|
|
def guess_license(srctree):
|
||
|
|
import bb
|
||
|
|
md5sums = get_license_md5sums(tinfoil.config_data)
|
||
|
|
|
||
|
|
licenses = []
|
||
|
|
licspecs = ['LICENSE*', 'COPYING*', '*[Ll]icense*', 'LICENCE*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
|
||
|
|
licfiles = []
|
||
|
|
for root, dirs, files in os.walk(srctree):
|
||
|
|
for fn in files:
|
||
|
|
for spec in licspecs:
|
||
|
|
if fnmatch.fnmatch(fn, spec):
|
||
|
|
licfiles.append(os.path.join(root, fn))
|
||
|
|
for licfile in licfiles:
|
||
|
|
md5value = bb.utils.md5_file(licfile)
|
||
|
|
license = md5sums.get(md5value, 'Unknown')
|
||
|
|
licenses.append((license, os.path.relpath(licfile, srctree), md5value))
|
||
|
|
|
||
|
|
# FIXME should we grab at least one source file with a license header and add that too?
|
||
|
|
|
||
|
|
return licenses
|
||
|
|
|
||
|
|
def read_pkgconfig_provides(d):
|
||
|
|
pkgdatadir = d.getVar('PKGDATA_DIR', True)
|
||
|
|
pkgmap = {}
|
||
|
|
for fn in glob.glob(os.path.join(pkgdatadir, 'shlibs2', '*.pclist')):
|
||
|
|
with open(fn, 'r') as f:
|
||
|
|
for line in f:
|
||
|
|
pkgmap[os.path.basename(line.rstrip())] = os.path.splitext(os.path.basename(fn))[0]
|
||
|
|
recipemap = {}
|
||
|
|
for pc, pkg in pkgmap.iteritems():
|
||
|
|
pkgdatafile = os.path.join(pkgdatadir, 'runtime', pkg)
|
||
|
|
if os.path.exists(pkgdatafile):
|
||
|
|
with open(pkgdatafile, 'r') as f:
|
||
|
|
for line in f:
|
||
|
|
if line.startswith('PN: '):
|
||
|
|
recipemap[pc] = line.split(':', 1)[1].strip()
|
||
|
|
return recipemap
|
||
|
|
|
||
|
|
def convert_pkginfo(pkginfofile):
|
||
|
|
values = {}
|
||
|
|
with open(pkginfofile, 'r') as f:
|
||
|
|
indesc = False
|
||
|
|
for line in f:
|
||
|
|
if indesc:
|
||
|
|
if line.strip():
|
||
|
|
values['DESCRIPTION'] += ' ' + line.strip()
|
||
|
|
else:
|
||
|
|
indesc = False
|
||
|
|
else:
|
||
|
|
splitline = line.split(': ', 1)
|
||
|
|
key = line[0]
|
||
|
|
value = line[1]
|
||
|
|
if key == 'LICENSE':
|
||
|
|
for dep in value.split(','):
|
||
|
|
dep = dep.split()[0]
|
||
|
|
mapped = depmap.get(dep, '')
|
||
|
|
if mapped:
|
||
|
|
depends.append(mapped)
|
||
|
|
elif key == 'License':
|
||
|
|
values['LICENSE'] = value
|
||
|
|
elif key == 'Summary':
|
||
|
|
values['SUMMARY'] = value
|
||
|
|
elif key == 'Description':
|
||
|
|
values['DESCRIPTION'] = value
|
||
|
|
indesc = True
|
||
|
|
return values
|
||
|
|
|
||
|
|
def convert_debian(debpath):
|
||
|
|
# FIXME extend this mapping - perhaps use distro_alias.inc?
|
||
|
|
depmap = {'libz-dev': 'zlib'}
|
||
|
|
|
||
|
|
values = {}
|
||
|
|
depends = []
|
||
|
|
with open(os.path.join(debpath, 'control')) as f:
|
||
|
|
indesc = False
|
||
|
|
for line in f:
|
||
|
|
if indesc:
|
||
|
|
if line.strip():
|
||
|
|
if line.startswith(' This package contains'):
|
||
|
|
indesc = False
|
||
|
|
else:
|
||
|
|
values['DESCRIPTION'] += ' ' + line.strip()
|
||
|
|
else:
|
||
|
|
indesc = False
|
||
|
|
else:
|
||
|
|
splitline = line.split(':', 1)
|
||
|
|
key = line[0]
|
||
|
|
value = line[1]
|
||
|
|
if key == 'Build-Depends':
|
||
|
|
for dep in value.split(','):
|
||
|
|
dep = dep.split()[0]
|
||
|
|
mapped = depmap.get(dep, '')
|
||
|
|
if mapped:
|
||
|
|
depends.append(mapped)
|
||
|
|
elif key == 'Section':
|
||
|
|
values['SECTION'] = value
|
||
|
|
elif key == 'Description':
|
||
|
|
values['SUMMARY'] = value
|
||
|
|
indesc = True
|
||
|
|
|
||
|
|
if depends:
|
||
|
|
values['DEPENDS'] = ' '.join(depends)
|
||
|
|
|
||
|
|
return values
|
||
|
|
|
||
|
|
|
||
|
|
def register_command(subparsers):
|
||
|
|
parser_create = subparsers.add_parser('create',
|
||
|
|
help='Create a new recipe',
|
||
|
|
description='Creates a new recipe from a source tree')
|
||
|
|
parser_create.add_argument('source', help='Path or URL to source')
|
||
|
|
parser_create.add_argument('-o', '--outfile', help='Specify filename for recipe to create', required=True)
|
||
|
|
parser_create.add_argument('-m', '--machine', help='Make recipe machine-specific as opposed to architecture-specific', action='store_true')
|
||
|
|
parser_create.add_argument('-x', '--extract-to', metavar='EXTRACTPATH', help='Assuming source is a URL, fetch it and extract it to the directory specified as %(metavar)s')
|
||
|
|
parser_create.set_defaults(func=create_recipe)
|
||
|
|
|