diff options
157 files changed, 7766 insertions, 2767 deletions
@@ -81,7 +81,6 @@ install: cd "$(srcdir)/cnf"; \ install -m$(INSMODE) $(PORTAGE_CONFDIR_FILES) \ "$(DESTDIR)$(portage_confdir)"; \ - ln -sf "..$(portage_confdir)/make.globals" "$(DESTDIR)$(sysconfdir)"; \ install -m$(INSMODE) "$(srcdir)/cnf/make.conf" \ "$(DESTDIR)$(portage_confdir)/make.conf.example"; \ \ @@ -10,8 +10,17 @@ portage-2.2 sonames change during upgrade or downgrade, and the @preserved-rebuild package set which rebuilds consumers of preserved libraries. +portage-2.1.11 +------------- + +* Add support for experimental EAPI "4-slot-abi". Refer to the corresponding + html documentation that is installed with USE=doc, and also to the emerge(1) + man page for information about the related --ignore-built-slot-abi-deps and + --rebuild-if-new-slot-abi options. + portage-2.1.10 ------------- + * Improve handling of repositories/overlays for cases in which the same ebuild is provided by multiple repositories but with different masking status, and support for repository constraints on atoms (atom::repo) in configuration diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 64dff1c1356..93e67ed5ae3 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -20,6 +20,18 @@ portage-2.2 /var/lib/portage/world, has been extended to include nested sets that may be listed /var/lib/portage/world_sets. +portage-2.1.11 +================================== +* User-defined package sets can now be created by placing files in the + /etc/portage/sets/ directory. Refer to the emerge(1) and portage(5) man + pages for more information. + +portage-2.1.10.61 +================================== +* FEATURES=config-protect-if-modified is now enabled by default. This causes + the CONFIG_PROTECT behavior to be skipped for files that have not been + modified since they were installed. + portage-2.1.10.27 ================================== * FEATURES=fixpackages is now enabled unconditionally. Set --package-moves=n diff --git a/bin/dohtml.py b/bin/dohtml.py index 122daf3f5ce..3e80ef5f6f4 100755 --- a/bin/dohtml.py +++ b/bin/dohtml.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # @@ -11,18 +11,18 @@ # # # Detailed usage: -# dohtml <list-of-files> -# - will install the files in the list of files (space-separated list) into -# /usr/share/doc/${PF}/html, provided the file ends in .htm, .html, .css, -# .js, ,gif, .jpeg, .jpg, or .png. +# dohtml <list-of-files> +# - will install the files in the list of files (space-separated list) into +# /usr/share/doc/${PF}/html, provided the file ends in .css, .gif, .htm, +# .html, .jpeg, .jpg, .js or .png. # dohtml -r <list-of-files-and-directories> -# - will do as 'dohtml', but recurse into all directories, as long as the +# - will do as 'dohtml', but recurse into all directories, as long as the # directory name is not CVS # dohtml -A jpe,java [-r] <list-of-files[-and-directories]> # - will do as 'dohtml' but add .jpe,.java (default filter list is # added to your list) # dohtml -a png,gif,html,htm [-r] <list-of-files[-and-directories]> -# - will do as 'dohtml' but filter on .png,.gif,.html,.htm (default filter +# - will do as 'dohtml' but filter on .png,.gif,.html,.htm (default filter # list is ignored) # dohtml -x CVS,SCCS,RCS -r <list-of-files-and-directories> # - will do as 'dohtml -r', but ignore directories named CVS, SCCS, RCS @@ -47,6 +47,10 @@ def eqawarn(lines): os.spawnlp(os.P_WAIT, "bash", "bash", "-c", cmd) skipped_directories = [] +skipped_files = [] +warn_on_skipped_files = os.environ.get("PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES") is not None +unwarned_skipped_extensions = os.environ.get("PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS", "").split() +unwarned_skipped_files = os.environ.get("PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES", "").split() def install(basename, dirname, options, prefix=""): fullpath = basename @@ -64,10 +68,12 @@ def install(basename, dirname, options, prefix=""): sys.stderr.write("!!! dohtml: %s does not exist\n" % fullpath) return False elif os.path.isfile(fullpath): - ext = os.path.splitext(basename)[1] - if (len(ext) and ext[1:] in options.allowed_exts) or basename in options.allowed_files: + ext = os.path.splitext(basename)[1][1:] + if ext in options.allowed_exts or basename in options.allowed_files: dodir(destdir) dofile(fullpath, destdir + "/" + basename) + elif warn_on_skipped_files and ext not in unwarned_skipped_extensions and basename not in unwarned_skipped_files: + skipped_files.append(fullpath) elif options.recurse and os.path.isdir(fullpath) and \ basename not in options.disallowed_dirs: for i in os.listdir(fullpath): @@ -88,20 +94,22 @@ class OptionsClass: self.PF = "" self.ED = "" self.DOCDESTTREE = "" - + if "PF" in os.environ: self.PF = os.environ["PF"] - if os.environ.get("EAPI", "0") in ("0", "1", "2"): + if "force-prefix" not in os.environ.get("FEATURES", "").split() and \ + os.environ.get("EAPI", "0") in ("0", "1", "2"): self.ED = os.environ.get("D", "") else: self.ED = os.environ.get("ED", "") if "_E_DOCDESTTREE_" in os.environ: self.DOCDESTTREE = os.environ["_E_DOCDESTTREE_"] - - self.allowed_exts = [ 'htm', 'html', 'css', 'js', - 'gif', 'jpeg', 'jpg', 'png' ] + + self.allowed_exts = ['css', 'gif', 'htm', 'html', 'jpeg', 'jpg', 'js', 'png'] + if os.environ.get("EAPI", "0") in ("4-python",): + self.allowed_exts += ['ico', 'svg', 'xhtml', 'xml'] self.allowed_files = [] - self.disallowed_dirs = [ 'CVS' ] + self.disallowed_dirs = ['CVS'] self.recurse = False self.verbose = False self.doc_prefix = "" @@ -126,7 +134,7 @@ def print_help(): def parse_args(): options = OptionsClass() args = [] - + x = 1 while x < len(sys.argv): arg = sys.argv[x] @@ -158,7 +166,7 @@ def parse_args(): else: args.append(sys.argv[x]) x += 1 - + return (options, args) def main(): @@ -167,20 +175,20 @@ def main(): if options.verbose: print("Allowed extensions:", options.allowed_exts) - print("Document prefix : '" + options.doc_prefix + "'") + print("Document prefix : '" + options.doc_prefix + "'") print("Allowed files :", options.allowed_files) success = False - + for x in args: basename = os.path.basename(x) dirname = os.path.dirname(x) success |= install(basename, dirname, options) - global skipped_directories for x in skipped_directories: - eqawarn(["QA Notice: dohtml on directory " + \ - "'%s' without recursion option" % x]) + eqawarn(["QA Notice: dohtml on directory '%s' without recursion option" % x]) + for x in skipped_files: + eqawarn(["dohtml: skipped file '%s'" % x]) if success: retcode = 0 diff --git a/bin/ebuild b/bin/ebuild index 82f0ff99002..65e5bef630c 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -1,9 +1,10 @@ #!/usr/bin/python -O -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +import platform import signal import sys # This block ensures that ^C interrupts are handled quietly. @@ -26,9 +27,16 @@ except KeyboardInterrupt: def debug_signal(signum, frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) import imp +import io import optparse import os @@ -165,7 +173,12 @@ if not os.path.exists(ebuild): ebuild_split = ebuild.split("/") cpv = "%s/%s" % (ebuild_split[-3], pf) -if not portage.catpkgsplit(cpv): +with io.open(_unicode_encode(ebuild, encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace') as f: + eapi = portage._parse_eapi_ebuild_head(f)[0] +if eapi is None: + eapi = "0" +if not portage.catpkgsplit(cpv, eapi=eapi): print("!!! %s does not follow correct package syntax." % (cpv)) sys.exit(1) diff --git a/bin/ebuild-helpers/ecompressdir b/bin/ebuild-helpers/ecompressdir index 17ecd80821b..6801a07d4b9 100755 --- a/bin/ebuild-helpers/ecompressdir +++ b/bin/ebuild-helpers/ecompressdir @@ -2,7 +2,7 @@ # Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/helper-functions.sh if [[ -z $1 ]] ; then helpers_die "${0##*/}: at least one argument needed" @@ -61,11 +61,11 @@ funk_up_dir() { local negate="" [[ ${act} == "compress" ]] && negate="!" + local ret=0 # first we act on all the files find "${dir}" -type f ${negate} -iname '*'${suffix} -print0 | ${XARGS} -0 ${binary} ((ret|=$?)) - find "${dir}" -type l -print0 | \ while read -r -d $'\0' brokenlink ; do [[ -e ${brokenlink} ]] && continue olddest=$(readlink "${brokenlink}") @@ -80,15 +80,24 @@ funk_up_dir() { skip_dir_dest=${T}/ecompress-skip/${actual_dir#${ED}}/${brokenlink%/*}/${olddest} fi [[ -e ${skip_dir_dest} ]] && continue - [[ ${act} == "compress" ]] \ - && newdest="${olddest}${suffix}" \ - || newdest="${olddest%${suffix}}" + if [[ ${act} == "compress" ]] ; then + newdest=${olddest}${suffix} + else + [[ ${olddest} == *${suffix} ]] || continue + newdest=${olddest%${suffix}} + fi + if [[ "${newdest}" == /* ]] ; then + [[ -f "${D}${newdest}" ]] || continue + else + [[ -f "${dir}/${brokenlink%/*}/${newdest}" ]] || continue + fi rm -f "${brokenlink}" [[ ${act} == "compress" ]] \ && ln -snf "${newdest}" "${brokenlink}${suffix}" \ || ln -snf "${newdest}" "${brokenlink%${suffix}}" ((ret|=$?)) - done + done < <(find "${dir}" -type l -print0) + return ${ret} } # _relocate_skip_dirs(srctree, dsttree) @@ -116,6 +125,16 @@ ret=0 rm -rf "${T}"/ecompress-skip +decompressors=( + ".Z" "gunzip -f" + ".gz" "gunzip -f" + ".bz2" "bunzip2 -f" + ".xz" "unxz -f" + ".lzma" "unxz -f" +) + +multijob_init + for dir in "$@" ; do dir=${dir#/} dir="${ED}${dir}" @@ -136,9 +155,21 @@ for dir in "$@" ; do find "${dir}" -type f -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f # not uncommon for packages to compress doc files themselves - funk_up_dir "decompress" ".Z" "gunzip -f" - funk_up_dir "decompress" ".gz" "gunzip -f" - funk_up_dir "decompress" ".bz2" "bunzip2 -f" + for (( d = 0; d < ${#decompressors[@]}; d += 2 )) ; do + # It's faster to parallelize at this stage than to try to + # parallelize the compressors. This is because the find|xargs + # ends up launching less compressors overall, so the overhead + # of forking children ends up dominating. + ( + multijob_child_init + funk_up_dir "decompress" "${decompressors[i]}" "${decompressors[i+1]}" + ) & + multijob_post_fork + : $(( ret |= $? )) + done + + multijob_finish + : $(( ret |= $? )) # forcibly break all hard links as some compressors whine about it find "${dir}" -type f -links +1 -exec env file="{}" sh -c \ @@ -148,6 +179,7 @@ for dir in "$@" ; do if [[ -n ${suffix} ]] ; then vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED}}" funk_up_dir "compress" "${suffix}" "ecompress" + : $(( ret |= $? )) fi # finally, restore the skipped stuff diff --git a/bin/ebuild-helpers/prepstrip b/bin/ebuild-helpers/prepstrip index daaa25250d8..fe5c1bc3213 100755 --- a/bin/ebuild-helpers/prepstrip +++ b/bin/ebuild-helpers/prepstrip @@ -1,8 +1,8 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/helper-functions.sh # avoid multiple calls to `has`. this creates things like: # FEATURES_foo=false @@ -19,7 +19,7 @@ exp_tf FEATURES compressdebug installsources nostrip splitdebug exp_tf RESTRICT binchecks installsources strip [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) EPREFIX= ED=${D} ;; esac + case "${EAPI}" in 0|1|2) EPREFIX= ED=${D} ;; esac banner=false SKIP_STRIP=false @@ -62,16 +62,24 @@ prepstrip_sources_dir=${EPREFIX}/usr/src/debug/${CATEGORY}/${PF} type -P debugedit >/dev/null && debugedit_found=true || debugedit_found=false debugedit_warned=false -unset ${!INODE_*} +multijob_init -inode_var_name() { - if [[ $USERLAND = BSD ]] ; then - stat -f 'INODE_%d_%i' "$1" +# Setup $T filesystem layout that we care about. +tmpdir="${T}/prepstrip" +rm -rf "${tmpdir}" +mkdir -p "${tmpdir}"/{inodes,splitdebug,sources} + +# Usage: inode_var_name: <file> +inode_file_link() { + echo -n "${tmpdir}/inodes/" + if [[ ${USERLAND} == "BSD" ]] ; then + stat -f '%i' "$1" else - stat -c 'INODE_%d_%i' "$1" + stat -c '%i' "$1" fi } +# Usage: save_elf_sources <elf> save_elf_sources() { ${FEATURES_installsources} || return 0 ${RESTRICT_installsources} && return 0 @@ -85,8 +93,7 @@ save_elf_sources() { fi local x=$1 - local inode=$(inode_var_name "$x") - [[ -n ${!inode} ]] && return 0 + [[ -f $(inode_file_link "${x}") ]] && return 0 # since we're editing the ELF here, we should recompute the build-id # (the -i flag below). save that output so we don't need to recompute @@ -94,10 +101,11 @@ save_elf_sources() { buildid=$(debugedit -i \ -b "${WORKDIR}" \ -d "${prepstrip_sources_dir}" \ - -l "${T}"/debug.sources \ + -l "${tmpdir}/sources/${x##*/}.${BASHPID}" \ "${x}") } +# Usage: save_elf_debug <elf> [splitdebug file] save_elf_debug() { ${FEATURES_splitdebug} || return 0 @@ -106,6 +114,7 @@ save_elf_debug() { # twice in this path) in order for gdb's debug-file-directory # lookup to work correctly. local x=$1 + local splitdebug=$2 local y=${ED}usr/lib/debug/${x:${#D}}.debug # dont save debug info twice @@ -113,13 +122,12 @@ save_elf_debug() { mkdir -p "${y%/*}" - local inode=$(inode_var_name "$x") - if [[ -n ${!inode} ]] ; then - ln "${ED}usr/lib/debug/${!inode:${#D}}.debug" "$y" + local inode=$(inode_file_link "${x}") + if [[ -f ${inode} ]] ; then + ln "${inode}" "${y}" else - eval $inode=\$x - if [[ -e ${T}/prepstrip.split.debug ]] ; then - mv "${T}"/prepstrip.split.debug "${y}" + if [[ -n ${splitdebug} ]] ; then + mv "${splitdebug}" "${y}" else local objcopy_flags="--only-keep-debug" ${FEATURES_compressdebug} && objcopy_flags+=" --compress-debug-sections" @@ -129,6 +137,7 @@ save_elf_debug() { local args="a-x,o-w" [[ -g ${x} || -u ${x} ]] && args+=",go-r" chmod ${args} "${y}" + ln "${y}" "${inode}" fi # if we don't already have build-id from debugedit, look it up @@ -146,6 +155,7 @@ save_elf_debug() { fi } +# Usage: process_elf <elf> process_elf() { local x=$1 strip_flags=${*:2} @@ -153,17 +163,34 @@ process_elf() { save_elf_sources "${x}" if ${strip_this} ; then + + # If two processes try to strip the same hardlink at the same + # time, it will cause one of them to lose the splitdebug info. + # So, use a lockfile to prevent interference (easily observed + # with dev-vcs/git which creates ~109 hardlinks to one file in + # /usr/libexec/git-core). + local lockfile=$(inode_file_link "${x}")_lockfile + if ! ln "${x}" "${lockfile}" ; then + while [[ -f ${lockfile} ]] ; do + sleep 1 + done + unset lockfile + fi + # see if we can split & strip at the same time if [[ -n ${SPLIT_STRIP_FLAGS} ]] ; then + local shortname="${x##*/}.debug" + local splitdebug="${tmpdir}/splitdebug/${shortname}.${BASHPID}" ${STRIP} ${strip_flags} \ - -f "${T}"/prepstrip.split.debug \ - -F "${x##*/}.debug" \ + -f "${splitdebug}" \ + -F "${shortname}" \ "${x}" - save_elf_debug "${x}" + save_elf_debug "${x}" "${splitdebug}" else save_elf_debug "${x}" ${STRIP} ${strip_flags} "${x}" fi + [[ -n ${lockfile} ]] && rm -f "${lockfile}" fi } @@ -171,28 +198,35 @@ process_elf() { # We want to log already stripped binaries, as this may be a QA violation. # They prevent us from getting the splitdebug data. if ! ${RESTRICT_binchecks} && ! ${RESTRICT_strip} ; then - log=$T/scanelf-already-stripped.log + # We need to do the non-stripped scan serially first before we turn around + # and start stripping the files ourselves. The log parsing can be done in + # parallel though. + log=${tmpdir}/scanelf-already-stripped.log + scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^${ED}##" > "${log}" + ( + multijob_child_init qa_var="QA_PRESTRIPPED_${ARCH/-/_}" [[ -n ${!qa_var} ]] && QA_PRESTRIPPED="${!qa_var}" - scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^${ED}##" > "$log" - if [[ -n $QA_PRESTRIPPED && -s $log && \ + if [[ -n ${QA_PRESTRIPPED} && -s ${log} && \ ${QA_STRICT_PRESTRIPPED-unset} = unset ]] ; then shopts=$- set -o noglob - for x in $QA_PRESTRIPPED ; do - sed -e "s#^${x#/}\$##" -i "$log" + for x in ${QA_PRESTRIPPED} ; do + sed -e "s#^${x#/}\$##" -i "${log}" done set +o noglob - set -$shopts + set -${shopts} fi - sed -e "/^\$/d" -e "s#^#/#" -i "$log" - if [[ -s $log ]] ; then + sed -e "/^\$/d" -e "s#^#/#" -i "${log}" + if [[ -s ${log} ]] ; then vecho -e "\n" eqawarn "QA Notice: Pre-stripped files found:" - eqawarn "$(<"$log")" + eqawarn "$(<"${log}")" else - rm -f "$log" + rm -f "${log}" fi + ) & + multijob_post_fork fi # Now we look for unstripped binaries. @@ -205,8 +239,10 @@ do banner=true fi - f=$(file "${x}") || continue - [[ -z ${f} ]] && continue + ( + multijob_child_init + f=$(file "${x}") || exit 0 + [[ -z ${f} ]] && exit 0 if ! ${SKIP_STRIP} ; then # The noglob funk is to support STRIP_MASK="/*/booga" and to keep @@ -253,16 +289,23 @@ do if ${was_not_writable} ; then chmod u-w "${x}" fi + ) & + multijob_post_fork done -if [[ -s ${T}/debug.sources ]] && \ +# With a bit more work, we could run the rsync processes below in +# parallel, but not sure that'd be an overall improvement. +multijob_finish + +cd "${tmpdir}"/sources/ && cat * > "${tmpdir}/debug.sources" 2>/dev/null +if [[ -s ${tmpdir}/debug.sources ]] && \ ${FEATURES_installsources} && \ ! ${RESTRICT_installsources} && \ ${debugedit_found} then vecho "installsources: rsyncing source files" [[ -d ${D}${prepstrip_sources_dir} ]] || mkdir -p "${D}${prepstrip_sources_dir}" - grep -zv '/<[^/>]*>$' "${T}"/debug.sources | \ + grep -zv '/<[^/>]*>$' "${tmpdir}"/debug.sources | \ (cd "${WORKDIR}"; LANG=C sort -z -u | \ rsync -tL0 --files-from=- "${WORKDIR}/" "${D}${prepstrip_sources_dir}/" ) @@ -271,6 +314,9 @@ then # https://bugzilla.redhat.com/show_bug.cgi?id=444310 while read -r -d $'\0' emptydir do - >> "$emptydir"/.keepdir + >> "${emptydir}"/.keepdir done < <(find "${D}${prepstrip_sources_dir}/" -type d -empty -print0) fi + +cd "${T}" +rm -rf "${tmpdir}" diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py index 29d4c23c602..3caf2d18569 100755 --- a/bin/ebuild-ipc.py +++ b/bin/ebuild-ipc.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # This is a helper which ebuild processes can use @@ -9,6 +9,7 @@ import errno import logging import os import pickle +import platform import select import signal import sys @@ -18,7 +19,13 @@ import traceback def debug_signal(signum, frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) # Avoid sandbox violations after python upgrade. pym_path = os.path.join(os.path.dirname( diff --git a/bin/ebuild.sh b/bin/ebuild.sh index b1d1df158f2..9829f68b3bb 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -375,7 +375,19 @@ source_all_bashrcs() { for x in "${path_array[@]}" ; do [ -f "$x/profile.bashrc" ] && qa_source "$x/profile.bashrc" done + fi + if [ -r "${PORTAGE_BASHRC}" ] ; then + if [ "$PORTAGE_DEBUG" != "1" ] || [ "${-/x/}" != "$-" ]; then + source "${PORTAGE_BASHRC}" + else + set -x + source "${PORTAGE_BASHRC}" + set +x + fi + fi + + if [[ $EBUILD_PHASE != depend ]] ; then # The user's bashrc is the ONLY non-portage bit of code that can # change shopts without a QA violation. for x in "${PM_EBUILD_HOOK_DIR}"/${CATEGORY}/{${PN},${PN}:${SLOT},${P},${PF}}; do @@ -394,16 +406,6 @@ source_all_bashrcs() { done fi - if [ -r "${PORTAGE_BASHRC}" ] ; then - if [ "$PORTAGE_DEBUG" != "1" ] || [ "${-/x/}" != "$-" ]; then - source "${PORTAGE_BASHRC}" - else - set -x - source "${PORTAGE_BASHRC}" - set +x - fi - fi - [ ! -z "${OCC}" ] && export CC="${OCC}" [ ! -z "${OCXX}" ] && export CXX="${OCXX}" } @@ -510,6 +512,10 @@ if ! has "$EBUILD_PHASE" clean cleanrm depend && \ [[ -n $EAPI ]] || EAPI=0 fi +if has "${EAPI:-0}" 4-python; then + shopt -s globstar +fi + if ! has "$EBUILD_PHASE" clean cleanrm ; then if [[ $EBUILD_PHASE = depend || ! -f $T/environment || \ -f $PORTAGE_BUILDDIR/.ebuild_changed ]] || \ @@ -528,7 +534,7 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then # In order to ensure correct interaction between ebuilds and # eclasses, they need to be unset before this process of # interaction begins. - unset DEPEND RDEPEND PDEPEND INHERITED IUSE REQUIRED_USE \ + unset EAPI DEPEND RDEPEND PDEPEND INHERITED IUSE REQUIRED_USE \ ECLASS E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND if [[ $PORTAGE_DEBUG != 1 || ${-/x/} != $- ]] ; then @@ -545,7 +551,10 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then rm "$PORTAGE_BUILDDIR/.ebuild_changed" fi - [[ -n $EAPI ]] || EAPI=0 + [ "${EAPI+set}" = set ] || EAPI=0 + + # export EAPI for helpers (especially since we unset it above) + export EAPI if has "$EAPI" 0 1 2 3 3_pre2 ; then export RDEPEND=${RDEPEND-${DEPEND}} @@ -664,8 +673,6 @@ if [[ $EBUILD_PHASE = depend ]] ; then PROPERTIES DEFINED_PHASES UNUSED_05 UNUSED_04 UNUSED_03 UNUSED_02 UNUSED_01" - [ -n "${EAPI}" ] || EAPI=0 - # The extra $(echo) commands remove newlines. if [ -n "${dbkey}" ] ; then > "${dbkey}" diff --git a/bin/egencache b/bin/egencache index b301115f2a3..a75a34172ce 100755 --- a/bin/egencache +++ b/bin/egencache @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2009-2011 Gentoo Foundation +# Copyright 2009-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -42,7 +42,7 @@ from portage.manifest import guessManifestFileType from portage.util import cmp_sort_key, writemsg_level from portage import cpv_getkey from portage.dep import Atom, isjustname -from portage.versions import pkgcmp, pkgsplit, vercmp +from portage.versions import pkgsplit, vercmp try: from xml.etree import ElementTree @@ -238,10 +238,15 @@ class GenCache(object): self._existing_nodes = set() - def _metadata_callback(self, cpv, repo_path, metadata, ebuild_hash): + def _metadata_callback(self, cpv, repo_path, metadata, + ebuild_hash, eapi_supported): self._existing_nodes.add(cpv) self._cp_missing.discard(cpv_getkey(cpv)) - if metadata is not None: + + # Since we're supposed to be able to efficiently obtain the + # EAPI from _parse_eapi_ebuild_head, we don't write cache + # entries for unsupported EAPIs. + if metadata is not None and eapi_supported: if metadata.get('EAPI') == '0': del metadata['EAPI'] for trg_cache in self._trg_caches: @@ -421,6 +426,9 @@ class GenCache(object): "committing target: %s\n" % (ce,), level=logging.ERROR, noiselevel=-1) + if hasattr(trg_cache, '_prune_empty_dirs'): + trg_cache._prune_empty_dirs() + class GenUseLocalDesc(object): def __init__(self, portdb, output=None, preserve_comments=False): @@ -565,7 +573,7 @@ class GenUseLocalDesc(object): return cmp_func(atomb.operator, atoma.operator) # Version matching elif atoma.cpv != atomb.cpv: - return pkgcmp(pkgsplit(atoma.cpv), pkgsplit(atomb.cpv)) + return vercmp(atoma.version, atomb.version) # Versions match, let's fallback to operator matching else: return cmp_func(ops.get(atoma.operator, -1), diff --git a/bin/emaint b/bin/emaint index 1bee0fe0b7a..bee46c40d20 100755 --- a/bin/emaint +++ b/bin/emaint @@ -1,16 +1,30 @@ #!/usr/bin/python -O -# vim: noet : +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program provides an interface to system health + checks and maintenance. +""" from __future__ import print_function -import errno -import re -import signal -import stat import sys -import textwrap -import time -from optparse import OptionParser, OptionValueError +import errno +# This block ensures that ^C interrupts are handled quietly. +try: + import signal + + def exithandler(signum,frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + sys.exit(1) + + signal.signal(signal.SIGINT, exithandler) + signal.signal(signal.SIGTERM, exithandler) + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +except KeyboardInterrupt: + sys.exit(1) try: import portage @@ -19,636 +33,13 @@ except ImportError: sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage -from portage import os -from portage.util import writemsg - -if sys.hexversion >= 0x3000000: - long = int - -class WorldHandler(object): - - short_desc = "Fix problems in the world file" - - def name(): - return "world" - name = staticmethod(name) - - def __init__(self): - self.invalid = [] - self.not_installed = [] - self.invalid_category = [] - self.okay = [] - from portage._sets import load_default_config - setconfig = load_default_config(portage.settings, - portage.db[portage.settings['EROOT']]) - self._sets = setconfig.getSets() - - def _check_world(self, onProgress): - categories = set(portage.settings.categories) - eroot = portage.settings['EROOT'] - self.world_file = os.path.join(eroot, portage.const.WORLD_FILE) - self.found = os.access(self.world_file, os.R_OK) - vardb = portage.db[eroot]["vartree"].dbapi - - from portage._sets import SETPREFIX - sets = self._sets - world_atoms = list(sets["selected"]) - maxval = len(world_atoms) - if onProgress: - onProgress(maxval, 0) - for i, atom in enumerate(world_atoms): - if not isinstance(atom, portage.dep.Atom): - if atom.startswith(SETPREFIX): - s = atom[len(SETPREFIX):] - if s in sets: - self.okay.append(atom) - else: - self.not_installed.append(atom) - else: - self.invalid.append(atom) - if onProgress: - onProgress(maxval, i+1) - continue - okay = True - if not vardb.match(atom): - self.not_installed.append(atom) - okay = False - if portage.catsplit(atom.cp)[0] not in categories: - self.invalid_category.append(atom) - okay = False - if okay: - self.okay.append(atom) - if onProgress: - onProgress(maxval, i+1) - - def check(self, onProgress=None): - self._check_world(onProgress) - errors = [] - if self.found: - errors += ["'%s' is not a valid atom" % x for x in self.invalid] - errors += ["'%s' is not installed" % x for x in self.not_installed] - errors += ["'%s' has a category that is not listed in /etc/portage/categories" % x for x in self.invalid_category] - else: - errors.append(self.world_file + " could not be opened for reading") - return errors - - def fix(self, onProgress=None): - world_set = self._sets["selected"] - world_set.lock() - try: - world_set.load() # maybe it's changed on disk - before = set(world_set) - self._check_world(onProgress) - after = set(self.okay) - errors = [] - if before != after: - try: - world_set.replace(self.okay) - except portage.exception.PortageException: - errors.append("%s could not be opened for writing" % \ - self.world_file) - return errors - finally: - world_set.unlock() - -class BinhostHandler(object): - - short_desc = "Generate a metadata index for binary packages" - - def name(): - return "binhost" - name = staticmethod(name) - - def __init__(self): - eroot = portage.settings['EROOT'] - self._bintree = portage.db[eroot]["bintree"] - self._bintree.populate() - self._pkgindex_file = self._bintree._pkgindex_file - self._pkgindex = self._bintree._load_pkgindex() - - def _need_update(self, cpv, data): - - if "MD5" not in data: - return True - - size = data.get("SIZE") - if size is None: - return True - - mtime = data.get("MTIME") - if mtime is None: - return True - - pkg_path = self._bintree.getname(cpv) - try: - s = os.lstat(pkg_path) - except OSError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): - raise - # We can't update the index for this one because - # it disappeared. - return False - - try: - if long(mtime) != s[stat.ST_MTIME]: - return True - if long(size) != long(s.st_size): - return True - except ValueError: - return True - - return False - - def check(self, onProgress=None): - missing = [] - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - maxval = len(cpv_all) - if onProgress: - onProgress(maxval, 0) - pkgindex = self._pkgindex - missing = [] - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - if onProgress: - onProgress(maxval, i+1) - errors = ["'%s' is not in Packages" % cpv for cpv in missing] - stale = set(metadata).difference(cpv_all) - for cpv in stale: - errors.append("'%s' is not in the repository" % cpv) - return errors - - def fix(self, onProgress=None): - bintree = self._bintree - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - missing = [] - maxval = 0 - if onProgress: - onProgress(maxval, 0) - pkgindex = self._pkgindex - missing = [] - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - - stale = set(metadata).difference(cpv_all) - if missing or stale: - from portage import locks - pkgindex_lock = locks.lockfile( - self._pkgindex_file, wantnewlockfile=1) - try: - # Repopulate with lock held. - bintree._populate() - cpv_all = self._bintree.dbapi.cpv_all() - cpv_all.sort() - - pkgindex = bintree._load_pkgindex() - self._pkgindex = pkgindex - - metadata = {} - for d in pkgindex.packages: - metadata[d["CPV"]] = d - - # Recount missing packages, with lock held. - del missing[:] - for i, cpv in enumerate(cpv_all): - d = metadata.get(cpv) - if not d or self._need_update(cpv, d): - missing.append(cpv) - - maxval = len(missing) - for i, cpv in enumerate(missing): - try: - metadata[cpv] = bintree._pkgindex_entry(cpv) - except portage.exception.InvalidDependString: - writemsg("!!! Invalid binary package: '%s'\n" % \ - bintree.getname(cpv), noiselevel=-1) - - if onProgress: - onProgress(maxval, i+1) - - for cpv in set(metadata).difference( - self._bintree.dbapi.cpv_all()): - del metadata[cpv] - - # We've updated the pkgindex, so set it to - # repopulate when necessary. - bintree.populated = False - - del pkgindex.packages[:] - pkgindex.packages.extend(metadata.values()) - from portage.util import atomic_ofstream - f = atomic_ofstream(self._pkgindex_file) - try: - self._pkgindex.write(f) - finally: - f.close() - finally: - locks.unlockfile(pkgindex_lock) - - if onProgress: - if maxval == 0: - maxval = 1 - onProgress(maxval, maxval) - return None - -class MoveHandler(object): - - def __init__(self, tree, porttree): - self._tree = tree - self._portdb = porttree.dbapi - self._update_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE"] - self._master_repo = \ - self._portdb.getRepositoryName(self._portdb.porttree_root) - - def _grab_global_updates(self): - from portage.update import grab_updates, parse_updates - retupdates = {} - errors = [] - - for repo_name in self._portdb.getRepositories(): - repo = self._portdb.getRepositoryPath(repo_name) - updpath = os.path.join(repo, "profiles", "updates") - if not os.path.isdir(updpath): - continue - - try: - rawupdates = grab_updates(updpath) - except portage.exception.DirectoryNotFound: - rawupdates = [] - upd_commands = [] - for mykey, mystat, mycontent in rawupdates: - commands, errors = parse_updates(mycontent) - upd_commands.extend(commands) - errors.extend(errors) - retupdates[repo_name] = upd_commands - - if self._master_repo in retupdates: - retupdates['DEFAULT'] = retupdates[self._master_repo] - - return retupdates, errors +from portage.emaint.main import emaint_main - def check(self, onProgress=None): - allupdates, errors = self._grab_global_updates() - # Matching packages and moving them is relatively fast, so the - # progress bar is updated in indeterminate mode. - match = self._tree.dbapi.match - aux_get = self._tree.dbapi.aux_get - if onProgress: - onProgress(0, 0) - for repo, updates in allupdates.items(): - if repo == 'DEFAULT': - continue - if not updates: - continue - - def repo_match(repository): - return repository == repo or \ - (repo == self._master_repo and \ - repository not in allupdates) - - for i, update_cmd in enumerate(updates): - if update_cmd[0] == "move": - origcp, newcp = update_cmd[1:] - for cpv in match(origcp): - if repo_match(aux_get(cpv, ["repository"])[0]): - errors.append("'%s' moved to '%s'" % (cpv, newcp)) - elif update_cmd[0] == "slotmove": - pkg, origslot, newslot = update_cmd[1:] - for cpv in match(pkg): - slot, prepo = aux_get(cpv, ["SLOT", "repository"]) - if slot == origslot and repo_match(prepo): - errors.append("'%s' slot moved from '%s' to '%s'" % \ - (cpv, origslot, newslot)) - if onProgress: - onProgress(0, 0) - - # Searching for updates in all the metadata is relatively slow, so this - # is where the progress bar comes out of indeterminate mode. - cpv_all = self._tree.dbapi.cpv_all() - cpv_all.sort() - maxval = len(cpv_all) - aux_update = self._tree.dbapi.aux_update - meta_keys = self._update_keys + ['repository'] - from portage.update import update_dbentries - if onProgress: - onProgress(maxval, 0) - for i, cpv in enumerate(cpv_all): - metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) - repository = metadata.pop('repository') - try: - updates = allupdates[repository] - except KeyError: - try: - updates = allupdates['DEFAULT'] - except KeyError: - continue - if not updates: - continue - metadata_updates = update_dbentries(updates, metadata) - if metadata_updates: - errors.append("'%s' has outdated metadata" % cpv) - if onProgress: - onProgress(maxval, i+1) - return errors - - def fix(self, onProgress=None): - allupdates, errors = self._grab_global_updates() - # Matching packages and moving them is relatively fast, so the - # progress bar is updated in indeterminate mode. - move = self._tree.dbapi.move_ent - slotmove = self._tree.dbapi.move_slot_ent - if onProgress: - onProgress(0, 0) - for repo, updates in allupdates.items(): - if repo == 'DEFAULT': - continue - if not updates: - continue - - def repo_match(repository): - return repository == repo or \ - (repo == self._master_repo and \ - repository not in allupdates) - - for i, update_cmd in enumerate(updates): - if update_cmd[0] == "move": - move(update_cmd, repo_match=repo_match) - elif update_cmd[0] == "slotmove": - slotmove(update_cmd, repo_match=repo_match) - if onProgress: - onProgress(0, 0) - - # Searching for updates in all the metadata is relatively slow, so this - # is where the progress bar comes out of indeterminate mode. - self._tree.dbapi.update_ents(allupdates, onProgress=onProgress) - return errors - -class MoveInstalled(MoveHandler): - - short_desc = "Perform package move updates for installed packages" - - def name(): - return "moveinst" - name = staticmethod(name) - def __init__(self): - eroot = portage.settings['EROOT'] - MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"]) - -class MoveBinary(MoveHandler): - - short_desc = "Perform package move updates for binary packages" - - def name(): - return "movebin" - name = staticmethod(name) - def __init__(self): - eroot = portage.settings['EROOT'] - MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree']) - -class VdbKeyHandler(object): - def name(): - return "vdbkeys" - name = staticmethod(name) - - def __init__(self): - self.list = portage.db[portage.settings["EROOT"]]["vartree"].dbapi.cpv_all() - self.missing = [] - self.keys = ["HOMEPAGE", "SRC_URI", "KEYWORDS", "DESCRIPTION"] - - for p in self.list: - mydir = os.path.join(portage.settings["EROOT"], portage.const.VDB_PATH, p)+os.sep - ismissing = True - for k in self.keys: - if os.path.exists(mydir+k): - ismissing = False - break - if ismissing: - self.missing.append(p) - - def check(self): - return ["%s has missing keys" % x for x in self.missing] - - def fix(self): - - errors = [] - - for p in self.missing: - mydir = os.path.join(portage.settings["EROOT"], portage.const.VDB_PATH, p)+os.sep - if not os.access(mydir+"environment.bz2", os.R_OK): - errors.append("Can't access %s" % (mydir+"environment.bz2")) - elif not os.access(mydir, os.W_OK): - errors.append("Can't create files in %s" % mydir) - else: - env = os.popen("bzip2 -dcq "+mydir+"environment.bz2", "r") - envlines = env.read().split("\n") - env.close() - for k in self.keys: - s = [l for l in envlines if l.startswith(k+"=")] - if len(s) > 1: - errors.append("multiple matches for %s found in %senvironment.bz2" % (k, mydir)) - elif len(s) == 0: - s = "" - else: - s = s[0].split("=",1)[1] - s = s.lstrip("$").strip("\'\"") - s = re.sub("(\\\\[nrt])+", " ", s) - s = " ".join(s.split()).strip() - if s != "": - try: - keyfile = open(mydir+os.sep+k, "w") - keyfile.write(s+"\n") - keyfile.close() - except (IOError, OSError) as e: - errors.append("Could not write %s, reason was: %s" % (mydir+k, e)) - - return errors - -class ProgressHandler(object): - def __init__(self): - self.curval = 0 - self.maxval = 0 - self.last_update = 0 - self.min_display_latency = 0.2 - - def onProgress(self, maxval, curval): - self.maxval = maxval - self.curval = curval - cur_time = time.time() - if cur_time - self.last_update >= self.min_display_latency: - self.last_update = cur_time - self.display() - - def display(self): - raise NotImplementedError(self) - -class CleanResume(object): - - short_desc = "Discard emerge --resume merge lists" - - def name(): - return "cleanresume" - name = staticmethod(name) - - def check(self, onProgress=None): - messages = [] - mtimedb = portage.mtimedb - resume_keys = ("resume", "resume_backup") - maxval = len(resume_keys) - if onProgress: - onProgress(maxval, 0) - for i, k in enumerate(resume_keys): - try: - d = mtimedb.get(k) - if d is None: - continue - if not isinstance(d, dict): - messages.append("unrecognized resume list: '%s'" % k) - continue - mergelist = d.get("mergelist") - if mergelist is None or not hasattr(mergelist, "__len__"): - messages.append("unrecognized resume list: '%s'" % k) - continue - messages.append("resume list '%s' contains %d packages" % \ - (k, len(mergelist))) - finally: - if onProgress: - onProgress(maxval, i+1) - return messages - - def fix(self, onProgress=None): - delete_count = 0 - mtimedb = portage.mtimedb - resume_keys = ("resume", "resume_backup") - maxval = len(resume_keys) - if onProgress: - onProgress(maxval, 0) - for i, k in enumerate(resume_keys): - try: - if mtimedb.pop(k, None) is not None: - delete_count += 1 - finally: - if onProgress: - onProgress(maxval, i+1) - if delete_count: - mtimedb.commit() - -def emaint_main(myargv): - - # Similar to emerge, emaint needs a default umask so that created - # files (such as the world file) have sane permissions. - os.umask(0o22) - - # TODO: Create a system that allows external modules to be added without - # the need for hard coding. - modules = { - "world" : WorldHandler, - "binhost":BinhostHandler, - "moveinst":MoveInstalled, - "movebin":MoveBinary, - "cleanresume":CleanResume - } - - module_names = list(modules) - module_names.sort() - module_names.insert(0, "all") - - def exclusive(option, *args, **kw): - var = kw.get("var", None) - if var is None: - raise ValueError("var not specified to exclusive()") - if getattr(parser, var, ""): - raise OptionValueError("%s and %s are exclusive options" % (getattr(parser, var), option)) - setattr(parser, var, str(option)) - - - usage = "usage: emaint [options] COMMAND" - - desc = "The emaint program provides an interface to system health " + \ - "checks and maintenance. See the emaint(1) man page " + \ - "for additional information about the following commands:" - - usage += "\n\n" - for line in textwrap.wrap(desc, 65): - usage += "%s\n" % line - usage += "\n" - usage += " %s" % "all".ljust(15) + \ - "Perform all supported commands\n" - for m in module_names[1:]: - usage += " %s%s\n" % (m.ljust(15), modules[m].short_desc) - - parser = OptionParser(usage=usage, version=portage.VERSION) - parser.add_option("-c", "--check", help="check for problems", - action="callback", callback=exclusive, callback_kwargs={"var":"action"}) - parser.add_option("-f", "--fix", help="attempt to fix problems", - action="callback", callback=exclusive, callback_kwargs={"var":"action"}) - parser.action = None - - - (options, args) = parser.parse_args(args=myargv) - if len(args) != 1: - parser.error("Incorrect number of arguments") - if args[0] not in module_names: - parser.error("%s target is not a known target" % args[0]) - - if parser.action: - action = parser.action - else: - print("Defaulting to --check") - action = "-c/--check" - - if args[0] == "all": - tasks = modules.values() - else: - tasks = [modules[args[0]]] - - - if action == "-c/--check": - status = "Checking %s for problems" - func = "check" - else: - status = "Attempting to fix %s" - func = "fix" - - isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() - for task in tasks: - print(status % task.name()) - inst = task() - onProgress = None - if isatty: - progressBar = portage.output.TermProgressBar() - progressHandler = ProgressHandler() - onProgress = progressHandler.onProgress - def display(): - progressBar.set(progressHandler.curval, progressHandler.maxval) - progressHandler.display = display - def sigwinch_handler(signum, frame): - lines, progressBar.term_columns = \ - portage.output.get_term_size() - signal.signal(signal.SIGWINCH, sigwinch_handler) - result = getattr(inst, func)(onProgress=onProgress) - if isatty: - # make sure the final progress is displayed - progressHandler.display() - print() - signal.signal(signal.SIGWINCH, signal.SIG_DFL) - if result: - print() - print("\n".join(result)) - print("\n") - - print("Finished") - -if __name__ == "__main__": +try: emaint_main(sys.argv[1:]) +except IOError as e: + if e.errno == errno.EACCES: + print("\nemaint: Need superuser access") + sys.exit(1) + else: + raise diff --git a/bin/emerge b/bin/emerge index 6f69244be90..a9a56432c23 100755 --- a/bin/emerge +++ b/bin/emerge @@ -1,9 +1,10 @@ #!/usr/bin/python -# Copyright 2006-2011 Gentoo Foundation +# Copyright 2006-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +import platform import signal import sys # This block ensures that ^C interrupts are handled quietly. @@ -26,7 +27,13 @@ except KeyboardInterrupt: def debug_signal(signum, frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) try: from _emerge.main import emerge_main diff --git a/bin/etc-update b/bin/etc-update index 1edc91fab70..d763c1f7389 100755 --- a/bin/etc-update +++ b/bin/etc-update @@ -190,6 +190,7 @@ parse_automode_flag() { parse_automode_flag -3 export mv_opts=" ${mv_opts} " mv_opts="${mv_opts// -i / }" + NONINTERACTIVE_MV=true ;; -3) input=0 @@ -600,6 +601,7 @@ declare title="Gentoo's etc-update tool!" PREEN=false SET_X=false VERBOSE=false +NONINTERACTIVE_MV=false while [[ -n $1 ]] ; do case $1 in -d|--debug) SET_X=true;; @@ -692,6 +694,11 @@ else fi fi +if ${NONINTERACTIVE_MV} ; then + export mv_opts=" ${mv_opts} " + mv_opts="${mv_opts// -i / }" +fi + if ${VERBOSE} ; then for v in ${portage_vars[@]} ${cfg_vars[@]} TMP SCAN_PATHS ; do echo "${v}=${!v}" diff --git a/bin/helper-functions.sh b/bin/helper-functions.sh new file mode 100644 index 00000000000..c7400fa4b2c --- /dev/null +++ b/bin/helper-functions.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Copyright 1999-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# For routines we want to use in ebuild-helpers/ but don't want to +# expose to the general ebuild environment. + +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh + +# +# API functions for doing parallel processing +# +numjobs() { + # Copied from eutils.eclass:makeopts_jobs() + local jobs=$(echo " ${MAKEOPTS} " | \ + sed -r -n 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p') + echo ${jobs:-1} +} + +multijob_init() { + # Setup a pipe for children to write their pids to when they finish. + mj_control_pipe=$(mktemp -t multijob.XXXXXX) + rm "${mj_control_pipe}" + mkfifo "${mj_control_pipe}" + redirect_alloc_fd mj_control_fd "${mj_control_pipe}" + rm -f "${mj_control_pipe}" + + # See how many children we can fork based on the user's settings. + mj_max_jobs=$(numjobs) + mj_num_jobs=0 +} + +multijob_child_init() { + trap 'echo ${BASHPID} $? >&'${mj_control_fd} EXIT + trap 'exit 1' INT TERM +} + +multijob_finish_one() { + local pid ret + read -r -u ${mj_control_fd} pid ret + : $(( --mj_num_jobs )) + return ${ret} +} + +multijob_finish() { + local ret=0 + while [[ ${mj_num_jobs} -gt 0 ]] ; do + multijob_finish_one + : $(( ret |= $? )) + done + # Let bash clean up its internal child tracking state. + wait + return ${ret} +} + +multijob_post_fork() { + : $(( ++mj_num_jobs )) + if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then + multijob_finish_one + fi + return $? +} + +# @FUNCTION: redirect_alloc_fd +# @USAGE: <var> <file> [redirection] +# @DESCRIPTION: +# Find a free fd and redirect the specified file via it. Store the new +# fd in the specified variable. Useful for the cases where we don't care +# about the exact fd #. +redirect_alloc_fd() { + local var=$1 file=$2 redir=${3:-"<>"} + + if [[ $(( (BASH_VERSINFO[0] << 8) + BASH_VERSINFO[1] )) -ge $(( (4 << 8) + 1 )) ]] ; then + # Newer bash provides this functionality. + eval "exec {${var}}${redir}'${file}'" + else + # Need to provide the functionality ourselves. + local fd=10 + while :; do + # Make sure the fd isn't open. It could be a char device, + # or a symlink (possibly broken) to something else. + if [[ ! -e /dev/fd/${fd} ]] && [[ ! -L /dev/fd/${fd} ]] ; then + eval "exec ${fd}${redir}'${file}'" && break + fi + [[ ${fd} -gt 1024 ]] && die "redirect_alloc_fd failed" + : $(( ++fd )) + done + : $(( ${var} = fd )) + fi +} diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh index 98be41ec0dc..dbf988b283a 100644 --- a/bin/isolated-functions.sh +++ b/bin/isolated-functions.sh @@ -173,8 +173,8 @@ die() { | while read -r n ; do eerror " ${n#RETAIN-LEADING-SPACE}" ; done eerror fi - eerror "If you need support, post the output of 'emerge --info =$CATEGORY/$PF'," - eerror "the complete build log and the output of 'emerge -pqv =$CATEGORY/$PF'." + eerror "If you need support, post the output of \`emerge --info '=$CATEGORY/$PF'\`," + eerror "the complete build log and the output of \`emerge -pqv '=$CATEGORY/$PF'\`." if [[ -n ${EBUILD_OVERLAY_ECLASSES} ]] ; then eerror "This ebuild used the following eclasses from overlays:" local x @@ -216,8 +216,15 @@ die() { > "$PORTAGE_BUILDDIR/.die_hooks" fi - [[ -n ${PORTAGE_LOG_FILE} ]] \ - && eerror "The complete build log is located at '${PORTAGE_LOG_FILE}'." + if [[ -n ${PORTAGE_LOG_FILE} ]] ; then + eerror "The complete build log is located at '${PORTAGE_LOG_FILE}'." + if [[ ${PORTAGE_LOG_FILE} != ${T}/* ]] ; then + # Display path to symlink in ${T}, as requested in bug #412865. + local log_ext=log + [[ ${PORTAGE_LOG_FILE} != *.log ]] && log_ext+=.${PORTAGE_LOG_FILE##*.} + eerror "For convenience, a symlink to the build log is located at '${T}/build.${log_ext}'." + fi + fi if [ -f "${T}/environment" ] ; then eerror "The ebuild environment file is located at '${T}/environment'." elif [ -d "${T}" ] ; then @@ -227,6 +234,7 @@ die() { } > "${T}/die.env" eerror "The ebuild environment file is located at '${T}/die.env'." fi + eerror "Working directory: '$(pwd)'" eerror "S: '${S}'" [[ -n $PORTAGE_EBUILD_EXIT_FILE ]] && > "$PORTAGE_EBUILD_EXIT_FILE" diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index b08389714cc..9eec8bb691f 100755 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -150,7 +150,7 @@ prepcompress() { install_qa_check() { local f i qa_var x [[ " ${FEATURES} " == *" force-prefix "* ]] || \ - case "$EAPI" in 0|1|2) local ED=${D} ;; esac + case "$EAPI" in 0|1|2) local EPREFIX= ED=${D} ;; esac cd "${ED}" || die "cd failed" @@ -433,43 +433,6 @@ install_qa_check() { fi fi - # Save NEEDED information after removing self-contained providers - rm -f "$PORTAGE_BUILDDIR"/build-info/NEEDED{,.ELF.2} - scanelf -qyRF '%a;%p;%S;%r;%n' "${D}" | { while IFS= read -r l; do - arch=${l%%;*}; l=${l#*;} - obj="/${l%%;*}"; l=${l#*;} - soname=${l%%;*}; l=${l#*;} - rpath=${l%%;*}; l=${l#*;}; [ "${rpath}" = " - " ] && rpath="" - needed=${l%%;*}; l=${l#*;} - if [ -z "${rpath}" -o -n "${rpath//*ORIGIN*}" ]; then - # object doesn't contain $ORIGIN in its runpath attribute - echo "${obj} ${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED - echo "${arch:3};${obj};${soname};${rpath};${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2 - else - dir=${obj%/*} - # replace $ORIGIN with the dirname of the current object for the lookup - opath=$(echo :${rpath}: | sed -e "s#.*:\(.*\)\$ORIGIN\(.*\):.*#\1${dir}\2#") - sneeded=$(echo ${needed} | tr , ' ') - rneeded="" - for lib in ${sneeded}; do - found=0 - for path in ${opath//:/ }; do - [ -e "${D}/${path}/${lib}" ] && found=1 && break - done - [ "${found}" -eq 0 ] && rneeded="${rneeded},${lib}" - done - rneeded=${rneeded:1} - if [ -n "${rneeded}" ]; then - echo "${obj} ${rneeded}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED - echo "${arch:3};${obj};${soname};${rpath};${rneeded}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2 - fi - fi - done } - - [ -n "${QA_SONAME_NO_SYMLINK}" ] && \ - echo "${QA_SONAME_NO_SYMLINK}" > \ - "${PORTAGE_BUILDDIR}"/build-info/QA_SONAME_NO_SYMLINK - if [[ ${insecure_rpath} -eq 1 ]] ; then die "Aborting due to serious QA concerns with RUNPATH/RPATH" elif [[ -n ${die_msg} ]] && has stricter ${FEATURES} ; then @@ -547,6 +510,34 @@ install_qa_check() { PORTAGE_QUIET=${tmp_quiet} fi + # Create NEEDED.ELF.2 regardless of RESTRICT=binchecks, since this info is + # too useful not to have (it's required for things like preserve-libs), and + # it's tempting for ebuild authors to set RESTRICT=binchecks for packages + # containing pre-built binaries. + if type -P scanelf > /dev/null ; then + # Save NEEDED information after removing self-contained providers + rm -f "$PORTAGE_BUILDDIR"/build-info/NEEDED{,.ELF.2} + scanelf -qyRF '%a;%p;%S;%r;%n' "${D}" | { while IFS= read -r l; do + arch=${l%%;*}; l=${l#*;} + obj="/${l%%;*}"; l=${l#*;} + soname=${l%%;*}; l=${l#*;} + rpath=${l%%;*}; l=${l#*;}; [ "${rpath}" = " - " ] && rpath="" + needed=${l%%;*}; l=${l#*;} + echo "${obj} ${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED + echo "${arch:3};${obj};${soname};${rpath};${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2 + done } + + [ -n "${QA_SONAME_NO_SYMLINK}" ] && \ + echo "${QA_SONAME_NO_SYMLINK}" > \ + "${PORTAGE_BUILDDIR}"/build-info/QA_SONAME_NO_SYMLINK + + if has binchecks ${RESTRICT} && \ + [ -s "${PORTAGE_BUILDDIR}/build-info/NEEDED.ELF.2" ] ; then + eqawarn "QA Notice: RESTRICT=binchecks prevented checks on these ELF files:" + eqawarn "$(while read -r x; do x=${x#*;} ; x=${x%%;*} ; echo "${x#${EPREFIX}}" ; done < "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2)" + fi + fi + local unsafe_files=$(find "${ED}" -type f '(' -perm -2002 -o -perm -4002 ')' | sed -e "s:^${ED}:/:") if [[ -n ${unsafe_files} ]] ; then eqawarn "QA Notice: Unsafe files detected (set*id and world writable)" @@ -782,16 +773,15 @@ install_qa_check() { fi if [[ ${abort} == "yes" ]] ; then if [[ $gentoo_bug = yes || $always_overflow = yes ]] ; then - die "install aborted due to" \ - "severe warnings shown above" + die "install aborted due to severe warnings shown above" else echo "Please do not file a Gentoo bug and instead" \ "report the above QA issues directly to the upstream" \ "developers of this software." | fmt -w 70 | \ while read -r line ; do eqawarn "${line}" ; done eqawarn "Homepage: ${HOMEPAGE}" - has stricter ${FEATURES} && die "install aborted due to" \ - "severe warnings shown above" + has stricter ${FEATURES} && \ + die "install aborted due to severe warnings shown above" fi fi fi @@ -1095,13 +1085,15 @@ preinst_selinux_labels() { # SELinux file labeling (needs to always be last in dyn_preinst) # only attempt to label if setfiles is executable # and 'context' is available on selinuxfs. - if [ -f /selinux/context -a -x /usr/sbin/setfiles -a -x /usr/sbin/selinuxconfig ]; then + if [ -f /selinux/context -o -f /sys/fs/selinux/context ] && \ + [ -x /usr/sbin/setfiles -a -x /usr/sbin/selinuxconfig ]; then vecho ">>> Setting SELinux security labels" ( eval "$(/usr/sbin/selinuxconfig)" || \ die "Failed to determine SELinux policy paths."; - addwrite /selinux/context; + addwrite /selinux/context + addwrite /sys/fs/selinux/context /usr/sbin/setfiles "${file_contexts_path}" -r "${D}" "${D}" ) || die "Failed to set SELinux security labels." diff --git a/bin/portageq b/bin/portageq index fcdb9d9631b..d9abb0bad73 100755 --- a/bin/portageq +++ b/bin/portageq @@ -86,7 +86,7 @@ def has_version(argv): """ if (len(argv) < 2): print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 warnings = [] @@ -116,11 +116,11 @@ def has_version(argv): try: mylist = portage.db[argv[0]]["vartree"].dbapi.match(atom) if mylist: - sys.exit(0) + return 0 else: - sys.exit(1) + return 1 except KeyError: - sys.exit(1) + return 1 except portage.exception.InvalidAtom: portage.writemsg("ERROR: Invalid atom: '%s'\n" % argv[1], noiselevel=-1) @@ -134,7 +134,7 @@ def best_version(argv): """ if (len(argv) < 2): print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 warnings = [] @@ -165,7 +165,7 @@ def best_version(argv): mylist = portage.db[argv[0]]["vartree"].dbapi.match(atom) print(portage.best(mylist)) except KeyError: - sys.exit(1) + return 1 best_version.uses_root = True @@ -175,19 +175,19 @@ def mass_best_version(argv): """ if (len(argv) < 2): print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 try: for pack in argv[1:]: mylist=portage.db[argv[0]]["vartree"].dbapi.match(pack) print(pack+":"+portage.best(mylist)) except KeyError: - sys.exit(1) + return 1 mass_best_version.uses_root = True def metadata(argv): if (len(argv) < 4): print("ERROR: insufficient parameters!", file=sys.stderr) - sys.exit(2) + return 2 eroot, pkgtype, pkgspec = argv[0:3] metakeys = argv[3:] @@ -197,7 +197,7 @@ def metadata(argv): "installed":"vartree"} if pkgtype not in type_map: print("Unrecognized package type: '%s'" % pkgtype, file=sys.stderr) - sys.exit(1) + return 1 trees = portage.db repo = portage.dep.dep_getrepo(pkgspec) pkgspec = portage.dep.remove_slot(pkgspec) @@ -207,7 +207,7 @@ def metadata(argv): writemsg_stdout(''.join('%s\n' % x for x in values), noiselevel=-1) except KeyError: print("Package not found: '%s'" % pkgspec, file=sys.stderr) - sys.exit(1) + return 1 metadata.__doc__ = """ <eroot> <pkgtype> <category/package> [<key>]+ @@ -504,23 +504,37 @@ def best_visible(argv): writemsg_stdout("%s\n" % (pkg.cpv,), noiselevel=-1) return os.EX_OK + # No package found, write out an empty line. + writemsg_stdout("\n", noiselevel=-1) + return 1 best_visible.uses_root = True def mass_best_visible(argv): - """<eroot> [<category/package>]+ + """<root> [<type>] [<category/package>]+ Returns category/package-version (without .ebuild). + The pkgtype argument defaults to "ebuild" if unspecified, + otherwise it must be one of ebuild, binary, or installed. """ + type_map = { + "ebuild":"porttree", + "binary":"bintree", + "installed":"vartree"} + if (len(argv) < 2): print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 try: - for pack in argv[1:]: - mylist=portage.db[argv[0]]["porttree"].dbapi.match(pack) - print(pack+":"+portage.best(mylist)) + root = argv.pop(0) + pkgtype = "ebuild" + if argv[0] in type_map: + pkgtype = argv.pop(0) + for pack in argv: + writemsg_stdout("%s:" % pack, noiselevel=-1) + best_visible([root, pkgtype, pack]) except KeyError: - sys.exit(1) + return 1 mass_best_visible.uses_root = True @@ -549,7 +563,7 @@ def match(argv): """ if len(argv) != 2: print("ERROR: expected 2 parameters, got %d!" % len(argv)) - sys.exit(2) + return 2 root, atom = argv if not atom: atom = "*/*" @@ -569,17 +583,15 @@ def match(argv): require_metadata = atom.slot or atom.repo for cpv in vardb.cpv_all(): - if not portage.dep.extended_cp_match( - atom.cp, portage.cpv_getkey(cpv)): + if not portage.match_from_list(atom, [cpv]): continue if require_metadata: - slot, repo = vardb.aux_get(cpv, ["SLOT", "repository"]) - - if atom.slot is not None and atom.slot != slot: + try: + cpv = vardb._pkg_str(cpv, atom.repo) + except (KeyError, portage.exception.InvalidData): continue - - if atom.repo is not None and atom.repo != repo: + if not portage.match_from_list(atom, [cpv]): continue results.append(cpv) @@ -704,7 +716,7 @@ def envvar(argv): if len(argv) == 0: print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 for arg in argv: if verbose: @@ -718,7 +730,7 @@ def get_repos(argv): """ if len(argv) < 1: print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 print(" ".join(portage.db[argv[0]]["porttree"].dbapi.getRepositories())) get_repos.uses_root = True @@ -729,7 +741,7 @@ def get_repo_path(argv): """ if len(argv) < 2: print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 for arg in argv[1:]: path = portage.db[argv[0]]["porttree"].dbapi.getRepositoryPath(arg) if path is None: @@ -747,7 +759,7 @@ def list_preserved_libs(argv): if len(argv) != 1: print("ERROR: wrong number of arguments") - sys.exit(2) + return 2 mylibs = portage.db[argv[0]]["vartree"].dbapi._plib_registry.getPreservedLibs() rValue = 1 msg = [] diff --git a/bin/quickpkg b/bin/quickpkg index d908c03469e..76259c5c1d5 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -21,7 +21,7 @@ except ImportError: from portage import os from portage import xpak from portage.dbapi.dep_expand import dep_expand -from portage.dep import Atom, extended_cp_match, use_reduce +from portage.dep import Atom, use_reduce from portage.exception import (AmbiguousPackageName, InvalidAtom, InvalidData, InvalidDependString, PackageSetNotFound, PermissionDenied) from portage.util import ConfigProtect, ensure_dirs, shlex_split @@ -68,11 +68,14 @@ def quickpkg_atom(options, infos, arg, eout): bintree.prevent_collision(cpv) dblnk = vardb._dblink(cpv) have_lock = False - try: - dblnk.lockdb() - have_lock = True - except PermissionDenied: - pass + + if "__PORTAGE_INHERIT_VARDB_LOCK" not in settings: + try: + dblnk.lockdb() + have_lock = True + except PermissionDenied: + pass + try: if not dblnk.exists(): # unmerged by a concurrent process @@ -201,16 +204,15 @@ def quickpkg_extended_atom(options, infos, atom, eout): atoms.append(cpv_atom) continue - if not extended_cp_match(atom.cp, cpv_atom.cp): + if not portage.match_from_list(atom, [cpv]): continue if require_metadata: - slot, repo = vardb.aux_get(cpv, ["SLOT", "repository"]) - - if atom.slot and atom.slot != slot: + try: + cpv = vardb._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): continue - - if atom.repo and atom.repo != repo: + if not portage.match_from_list(atom, [cpv]): continue atoms.append(cpv_atom) diff --git a/bin/repoman b/bin/repoman index 076d4042ffc..795c7ce7712 100755 --- a/bin/repoman +++ b/bin/repoman @@ -45,7 +45,11 @@ portage.dep._internal_warnings = True try: import xml.etree.ElementTree from xml.parsers.expat import ExpatError -except ImportError: +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 msg = ["Please enable python's \"xml\" USE flag in order to use repoman."] from portage.output import EOutput out = EOutput() @@ -75,7 +79,8 @@ from portage.process import find_binary, spawn from portage.output import bold, create_color_func, \ green, nocolor, red from portage.output import ConsoleStyleFile, StyleWriter -from portage.util import cmp_sort_key, writemsg_level +from portage.util import writemsg_level +from portage.util._desktop_entry import validate_desktop_entry from portage.package.ebuild.digestgen import digestgen from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use @@ -314,8 +319,9 @@ qahelp={ "file.size.fatal":"Files in the files directory must be under 60 KiB", "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars, "file.UTF8":"File is not UTF8 compliant", - "inherit.autotools":"Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf", "inherit.deprecated":"Ebuild inherits a deprecated eclass", + "inherit.missing":"Ebuild uses functions from an eclass but does not inherit it", + "inherit.unused":"Ebuild inherits an eclass but does not use it", "java.eclassesnotused":"With virtual/jdk in DEPEND you must inherit a java eclass", "wxwidgets.eclassnotused":"Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass", "KEYWORDS.dropped":"Ebuilds that appear to have dropped KEYWORDS for some arch", @@ -326,7 +332,7 @@ qahelp={ "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable", "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable", "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len, - "EAPI.definition":"EAPI is defined after an inherit call (must be defined before)", + "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)", "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI", "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI", "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)", @@ -381,13 +387,13 @@ qahelp={ "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully", "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style", "ebuild.badheader":"This ebuild has a malformed header", - "eprefixify.defined":"The ebuild uses eprefixify, but does not inherit the prefix eclass", "manifest.bad":"Manifest has missing or incorrect digests", "metadata.missing":"Missing metadata.xml files", "metadata.bad":"Bad metadata.xml files", "metadata.warning":"Warnings in metadata.xml files", "portage.internal":"The ebuild uses an internal Portage function", "virtual.oldstyle":"The ebuild PROVIDEs an old-style virtual (see GLEP 37)", + "virtual.suspect":"Ebuild contains a package that usually should be pulled via virtual/, not directly.", "usage.obsolete":"The ebuild makes use of an obsolete construct", "upstream.workaround":"The ebuild works around an upstream bug, an upstream bug should be filed and tracked in bugs.gentoo.org" } @@ -419,12 +425,13 @@ qawarnings = set(( "PDEPEND.suspect", "RDEPEND.implicit", "RDEPEND.suspect", +"virtual.suspect", "RESTRICT.invalid", "ebuild.minorsyn", "ebuild.badheader", "ebuild.patches", "file.size", -"inherit.autotools", +"inherit.unused", "inherit.deprecated", "java.eclassesnotused", "wxwidgets.eclassnotused", @@ -436,6 +443,10 @@ qawarnings = set(( "LIVEVCS.unmasked", )) +if portage.const._ENABLE_INHERIT_CHECK: + # This is experimental, so it's non-fatal. + qawarnings.add("inherit.missing") + non_ascii_re = re.compile(r'[^\x00-\x7f]') missingvars = ["KEYWORDS", "LICENSE", "DESCRIPTION", "HOMEPAGE"] @@ -481,7 +492,10 @@ suspect_rdepend = frozenset([ "dev-util/gtk-doc-am", "dev-util/intltool", "dev-util/jam", + "dev-util/pkg-config-lite", + "dev-util/pkgconf", "dev-util/pkgconfig", + "dev-util/pkgconfig-openbsd", "dev-util/scons", "dev-util/unifdef", "dev-util/yacc", @@ -496,10 +510,18 @@ suspect_rdepend = frozenset([ "sys-devel/m4", "sys-devel/pmake", "virtual/linux-sources", + "virtual/pkgconfig", "x11-misc/bdftopcf", "x11-misc/imake", ]) +suspect_virtual = { + "dev-util/pkg-config-lite":"virtual/pkgconfig", + "dev-util/pkgconf":"virtual/pkgconfig", + "dev-util/pkgconfig":"virtual/pkgconfig", + "dev-util/pkgconfig-openbsd":"virtual/pkgconfig", +} + metadata_dtd_uri = 'http://www.gentoo.org/dtd/metadata.dtd' # force refetch if the local copy creation time is older than this metadata_dtd_ctime_interval = 60 * 60 * 24 * 7 # 7 days @@ -1128,7 +1150,7 @@ if vcs == "cvs": if options.if_modified == "y": myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./") -if vcs == "svn": +elif vcs == "svn": with os.popen("svn status") as f: svnstatus = f.readlines() mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ] @@ -1186,6 +1208,9 @@ def vcs_new_changed(relative_path): have_pmasked = False have_dev_keywords = False dofail = 0 + +# NOTE: match-all caches are not shared due to potential +# differences between profiles in _get_implicit_iuse. arch_caches={} arch_xmatch_caches = {} shared_xmatch_caches = {"cp-list":{}} @@ -1348,17 +1373,6 @@ for x in effective_scanlist: pkgs[pf] = Package(cpv=cpv, metadata=myaux, root_config=root_config, type_name="ebuild") - # Sort ebuilds in ascending order for the KEYWORDS.dropped check. - pkgsplits = {} - for i in range(len(ebuildlist)): - ebuild_split = portage.pkgsplit(ebuildlist[i]) - pkgsplits[ebuild_split] = ebuildlist[i] - ebuildlist[i] = ebuild_split - ebuildlist.sort(key=cmp_sort_key(portage.pkgcmp)) - for i in range(len(ebuildlist)): - ebuildlist[i] = pkgsplits[ebuildlist[i]] - del pkgsplits - slot_keywords = {} if len(pkgs) != len(ebuildlist): @@ -1370,6 +1384,10 @@ for x in effective_scanlist: can_force = False continue + # Sort ebuilds in ascending order for the KEYWORDS.dropped check. + ebuildlist = sorted(pkgs.values()) + ebuildlist = [pkg.pf for pkg in ebuildlist] + for y in checkdirlist: m = disallowed_filename_chars_re.search(y.strip(os.sep)) if m is not None: @@ -1562,15 +1580,14 @@ for x in effective_scanlist: (checkdir, y, m.group(0))) if desktop_file_validate and desktop_pattern.match(y): - status, cmd_output = subprocess_getstatusoutput( - "'%s' '%s'" % (desktop_file_validate, full_path)) - if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK: + cmd_output = validate_desktop_entry(full_path) + if cmd_output: # Note: in the future we may want to grab the # warnings in addition to the errors. We're # just doing errors now since we don't want # to generate too much noise at first. error_re = re.compile(r'.*\s*error:\s*(.*)') - for line in cmd_output.splitlines(): + for line in cmd_output: error_match = error_re.match(line) if error_match is None: continue @@ -1794,7 +1811,7 @@ for x in effective_scanlist: Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should not be allowed to be marked stable """ - if live_ebuild: + if live_ebuild and repo_config.name == "gentoo": bad_stable_keywords = [] for keyword in keywords: if not keyword.startswith("~") and \ @@ -1876,6 +1893,15 @@ for x in effective_scanlist: is_blocker = atom.blocker + if catdir != "virtual": + if not is_blocker and \ + atom.cp in suspect_virtual: + stats['virtual.suspect'] += 1 + fails['virtual.suspect'].append( + relative_path + + ": %s: consider using '%s' instead of '%s'" % + (mytype, suspect_virtual[atom.cp], atom)) + if mytype == "DEPEND" and \ not is_blocker and \ not inherited_java_eclass and \ @@ -1962,12 +1988,12 @@ for x in effective_scanlist: #keyword checks myuse = myaux["KEYWORDS"].split() for mykey in myuse: - myskey=mykey[:] - if myskey[0]=="-": - myskey=myskey[1:] - if myskey[0]=="~": - myskey=myskey[1:] - if mykey!="-*": + if mykey not in ("-*", "*", "~*"): + myskey = mykey + if myskey[:1] == "-": + myskey = myskey[1:] + if myskey[:1] == "~": + myskey = myskey[1:] if myskey not in kwlist: stats["KEYWORDS.invalid"] += 1 fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey) @@ -2060,7 +2086,7 @@ for x in effective_scanlist: env=env) dep_settings.categories = repoman_settings.categories if options.without_mask: - dep_settings._mask_manager = \ + dep_settings._mask_manager_obj = \ copy.deepcopy(dep_settings._mask_manager) dep_settings._mask_manager._pmaskdict.clear() arch_caches[prof.sub_path] = dep_settings @@ -2539,6 +2565,11 @@ else: else: myupdates.append(changelog_path) + if options.ask and not options.pretend: + # regenerate Manifest for modified ChangeLog (bug #420735) + repoman_settings["O"] = checkdir + digestgen(mysettings=repoman_settings, myportdb=portdb) + if myautoadd: print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...") add_cmd = [vcs, "add"] @@ -2747,7 +2778,10 @@ else: for x in sorted(vcs_files_to_cps( chain(myupdates, myremoved, mymanifests))): repoman_settings["O"] = os.path.join(repodir, x) - gpgsign(os.path.join(repoman_settings["O"], "Manifest")) + manifest_path = os.path.join(repoman_settings["O"], "Manifest") + if not os.path.exists(manifest_path): + continue + gpgsign(manifest_path) except portage.exception.PortageException as e: portage.writemsg("!!! %s\n" % str(e)) portage.writemsg("!!! Disabled FEATURES='sign'\n") diff --git a/bin/save-ebuild-env.sh b/bin/save-ebuild-env.sh index 23e7aaa6a52..47a2acae54c 100644 --- a/bin/save-ebuild-env.sh +++ b/bin/save-ebuild-env.sh @@ -79,7 +79,11 @@ save_ebuild_env() { GOOD HILITE HOME \ LAST_E_CMD LAST_E_LEN LD_PRELOAD MISC_FUNCTIONS_ARGS MOPREFIX \ NOCOLOR NORMAL PKGDIR PKGUSE PKG_LOGDIR PKG_TMPDIR \ - PORTAGE_BASHRCS_SOURCED PORTAGE_COMPRESS_EXCLUDE_SUFFIXES \ + PORTAGE_BASHRCS_SOURCED PORTAGE_COMPRESS \ + PORTAGE_COMPRESS_EXCLUDE_SUFFIXES \ + PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS \ + PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES \ + PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES \ PORTAGE_NONFATAL PORTAGE_QUIET \ PORTAGE_SANDBOX_DENY PORTAGE_SANDBOX_PREDICT \ PORTAGE_SANDBOX_READ PORTAGE_SANDBOX_WRITE PREROOTPATH \ diff --git a/cnf/make.conf b/cnf/make.conf index 5ee1cd32ea9..ad2a5b7751f 100644 --- a/cnf/make.conf +++ b/cnf/make.conf @@ -33,7 +33,8 @@ # before reporting errors to developers. # # If your gcc supports it, you can add -frecord-gcc-switches to all of the -# following *FLAGS in order to enable *FLAGS ignorance checking for ebuilds. +# following *FLAGS in order to enable *FLAGS ignorance checking for ebuilds: +# CFLAGS, CXXFLAGS, FFLAGS, and FCFLAGS. # Note that this check is only enabled if every one of these variables contains # -frecord-gcc-switches, since otherwise the check could result in false # positive results. diff --git a/cnf/make.globals b/cnf/make.globals index 0f6f5416514..ada91f8f018 100644 --- a/cnf/make.globals +++ b/cnf/make.globals @@ -51,15 +51,20 @@ RESUMECOMMAND_SSH=${FETCHCOMMAND_SSH} FETCHCOMMAND_SFTP="bash -c \"x=\\\${2#sftp://} ; host=\\\${x%%/*} ; port=\\\${host##*:} ; host=\\\${host%:*} ; [[ \\\${host} = \\\${port} ]] && port=22 ; exec sftp -P \\\${port} \\\"\\\${host}:/\\\${x#*/}\\\" \\\"\\\$1\\\"\" sftp \"\${DISTDIR}/\${FILE}\" \"\${URI}\"" # Default user options -FEATURES="assume-digests binpkg-logs distlocks ebuild-locks - fixlafiles news parallel-fetch protect-owned +FEATURES="assume-digests binpkg-logs + config-protect-if-modified distlocks ebuild-locks + fixlafiles news parallel-fetch parse-eapi-ebuild-head protect-owned sandbox sfperms strict unknown-features-warn unmerge-logs unmerge-orphans userfetch" # Ignore file collisions in /lib/modules since files inside this directory # are never unmerged, and therefore collisions must be ignored in order for # FEATURES=protect-owned to operate smoothly in all cases. -COLLISION_IGNORE="/lib/modules" +# Ignore file collisions for unowned *.pyo and *.pyc files, this helps during +# transition from compiling python modules in live file system to compiling +# them in src_install() function. +COLLISION_IGNORE="/lib/modules/* *.py[co]" +UNINSTALL_IGNORE="/lib/modules/*" # Enable preserve-libs for testing with portage versions that support it. # This setting is commented out for portage versions that don't support it. @@ -123,7 +128,7 @@ PORTAGE_ELOG_MAILSUBJECT="[portage] ebuild log for \${PACKAGE} on \${HOST}" PORTAGE_ELOG_MAILFROM="portage@localhost" # Signing command used by repoman -PORTAGE_GPG_SIGNING_COMMAND="gpg --sign --clearsign --yes --default-key \"\${PORTAGE_GPG_KEY}\" --homedir \"\${PORTAGE_GPG_DIR}\" \"\${FILE}\"" +PORTAGE_GPG_SIGNING_COMMAND="gpg --sign --digest-algo SHA256 --clearsign --yes --default-key \"\${PORTAGE_GPG_KEY}\" --homedir \"\${PORTAGE_GPG_DIR}\" \"\${FILE}\"" # ***************************** # ** DO NOT EDIT THIS FILE ** diff --git a/doc/package/ebuild.docbook b/doc/package/ebuild.docbook index e771d3ae184..ba146ca994b 100644 --- a/doc/package/ebuild.docbook +++ b/doc/package/ebuild.docbook @@ -10,5 +10,6 @@ &package_ebuild_eapi_3; &package_ebuild_eapi_4; &package_ebuild_eapi_4_python; +&package_ebuild_eapi_4_slot_abi; </section> </chapter> diff --git a/doc/package/ebuild/eapi/4-python.docbook b/doc/package/ebuild/eapi/4-python.docbook index b4cbc1ad480..ec5fd83c44e 100644 --- a/doc/package/ebuild/eapi/4-python.docbook +++ b/doc/package/ebuild/eapi/4-python.docbook @@ -1,100 +1,122 @@ <section id='package-ebuild-eapi-4-python'> -<title>EAPI 4-python</title> -<para>Also see the -<ulink url="http://people.apache.org/~Arfrever/EAPI_4-python_Specification"> -official EAPI 4-python Specification</ulink>. -</para> -<section id='package-ebuild-eapi-4-python-helpers'> -<title>Helpers</title> -<section id='package-ebuild-eapi-4-python-helpers-banned-global-scope'> -<title>Helpers Banned in Global Scope</title> -<para> -<itemizedlist> -<listitem><para>diropts</para></listitem> -<listitem><para>docompress</para></listitem> -<listitem><para>exeopts</para></listitem> -<listitem><para>insopts</para></listitem> -<listitem><para>keepdir</para></listitem> -<listitem><para>libopts</para></listitem> -<listitem><para>use</para></listitem> -<listitem><para>use_enable</para></listitem> -<listitem><para>use_with</para></listitem> -<listitem><para>useq</para></listitem> -<listitem><para>usev</para></listitem> -</itemizedlist> -</para> -</section> -</section> -<section id='package-ebuild-eapi-4-python-use-flag-allow-period-characters'> -<title>Support for Period Characters in USE Flags</title> -<para> -The "." character is allowed in USE flags. -</para> -</section> -<section id='package-ebuild-eapi-4-python-repository-variable'> -<title>REPOSITORY Variable</title> -<para> -The new REPOSITORY variable is set in ebuild environment. This variable contains name of repository, which contains currently used ebuild. -</para> -</section> -<section id='package-ebuild-eapi-4-python-repository-dependencies'> -<title>Repository Dependencies</title> -<para> -Repository dependencies are supported in atoms in DEPEND, PDEPEND and RDEPEND and atoms passed to best_version and has_version functions. -Repository dependency is specified by two colons followed by repository name. -</para> -<table><title>Repository Dependency Examples</title> -<tgroup cols='1' align='left' > -<colspec colname='atom'/> -<thead> -<row> -<entry>Atom</entry> -</row> -</thead> -<tbody> -<row> -<entry>dev-lang/python::progress</entry> -</row> -<row> -<entry>>=dev-lang/python-3.2::progress</entry> -</row> -<row> -<entry>dev-lang/python:3.2::progress</entry> -</row> -<row> -<entry>dev-lang/python::progress[xml]</entry> -</row> -<row> -<entry>dev-lang/python:3.2::progress[xml]</entry> -</row> -</tbody> -</tgroup> -</table> -</section> -<section id='package-ebuild-eapi-4-python-repo-level-config'> -<title>Extended Repository-Level Configuration</title> -<para> -Repository-level configuration in ${repository}/profiles is supported for -the following files: -<itemizedlist> -<listitem><para>make.defaults</para></listitem> -<listitem><para>package.use</para></listitem> -<listitem><para>package.use.force</para></listitem> -<listitem><para>package.use.mask</para></listitem> -<listitem><para>use.force</para></listitem> -<listitem><para>use.mask</para></listitem> -</itemizedlist> -</para> -<para> -By default, the following files in ${repository}/profiles can be also directories: -<itemizedlist> -<listitem><para>package.mask</para></listitem> -<listitem><para>package.use</para></listitem> -<listitem><para>package.use.force</para></listitem> -<listitem><para>package.use.mask</para></listitem> -<listitem><para>use.force</para></listitem> -<listitem><para>use.mask</para></listitem> -</itemizedlist> -</para> -</section> + <title>EAPI 4-python</title> + <para> + Also see the <ulink url="http://people.apache.org/~Arfrever/EAPI_4-python_Specification">official EAPI 4-python Specification</ulink>. + </para> + <section id='package-ebuild-eapi-4-python-helpers'> + <title>Helpers</title> + <section id='package-ebuild-eapi-4-python-helpers-dohtml-extended-default-list-of-extensions'> + <title>Extended default list of extensions in dohtml</title> + <para> + dohtml by default additionally installs files with .ico, .svg, .xhtml and .xml extensions. + </para> + </section> + <section id='package-ebuild-eapi-4-python-helpers-banned-in-global-scope'> + <title>Helpers Banned in Global Scope</title> + <para> + <itemizedlist> + <listitem><para>diropts</para></listitem> + <listitem><para>docompress</para></listitem> + <listitem><para>exeopts</para></listitem> + <listitem><para>insopts</para></listitem> + <listitem><para>keepdir</para></listitem> + <listitem><para>libopts</para></listitem> + <listitem><para>use</para></listitem> + <listitem><para>use_enable</para></listitem> + <listitem><para>use_with</para></listitem> + <listitem><para>useq</para></listitem> + <listitem><para>usev</para></listitem> + </itemizedlist> + </para> + </section> + </section> + <section id='package-ebuild-eapi-4-python-metadata'> + <title>Metadata</title> + <section id='package-ebuild-eapi-4-python-metadata-package-names-allow-period-characters'> + <title>Support for Period Characters in Package Names</title> + <para> + The "." character is allowed in package names. + </para> + </section> + <section id='package-ebuild-eapi-4-python-metadata-use-flags-allow-period-characters'> + <title>Support for Period Characters in USE Flags</title> + <para> + The "." character is allowed in USE flags. + </para> + </section> + <section id='package-ebuild-eapi-4-python-metadata-repository-dependencies'> + <title>Repository Dependencies</title> + <para> + Repository dependencies are supported in atoms in DEPEND, PDEPEND and RDEPEND and atoms passed to best_version and has_version functions. + Repository dependency is specified by two colons followed by repository name. + </para> + <table><title>Repository Dependency Examples</title> + <tgroup cols='1' align='left'> + <colspec colname='atom'/> + <thead> + <row> + <entry>Atom</entry> + </row> + </thead> + <tbody> + <row> + <entry>dev-lang/python::progress</entry> + </row> + <row> + <entry>>=dev-lang/python-3.2::progress</entry> + </row> + <row> + <entry>dev-lang/python:3.2::progress</entry> + </row> + <row> + <entry>dev-lang/python::progress[xml]</entry> + </row> + <row> + <entry>dev-lang/python:3.2::progress[xml]</entry> + </row> + </tbody> + </tgroup> + </table> + </section> + </section> + <section id='package-ebuild-eapi-4-python-globstar'> + <title>globstar shell option enabled by default</title> + <para> + globstar shell option is enabled by default, which enables recursive expansion of ** pattern in pathname expansion context. + </para> + </section> + <section id='package-ebuild-eapi-4-python-variables'> + <title>Variables</title> + <section id='package-ebuild-eapi-4-python-variables-repository'> + <title>REPOSITORY Variable</title> + <para> + The new REPOSITORY variable is set in ebuild environment. This variable contains name of repository, which contains currently used ebuild. + </para> + </section> + </section> + <section id='package-ebuild-eapi-4-python-repo-level-config'> + <title>Extended Repository-Level Configuration</title> + <para> + Repository-level configuration in ${repository}/profiles is supported for the following files: + <itemizedlist> + <listitem><para>make.defaults</para></listitem> + <listitem><para>package.use</para></listitem> + <listitem><para>package.use.force</para></listitem> + <listitem><para>package.use.mask</para></listitem> + <listitem><para>use.force</para></listitem> + <listitem><para>use.mask</para></listitem> + </itemizedlist> + </para> + <para> + By default, the following files in ${repository}/profiles can be also directories: + <itemizedlist> + <listitem><para>package.mask</para></listitem> + <listitem><para>package.use</para></listitem> + <listitem><para>package.use.force</para></listitem> + <listitem><para>package.use.mask</para></listitem> + <listitem><para>use.force</para></listitem> + <listitem><para>use.mask</para></listitem> + </itemizedlist> + </para> + </section> </section> diff --git a/doc/package/ebuild/eapi/4-slot-abi.docbook b/doc/package/ebuild/eapi/4-slot-abi.docbook new file mode 100644 index 00000000000..696d0bf746c --- /dev/null +++ b/doc/package/ebuild/eapi/4-slot-abi.docbook @@ -0,0 +1,70 @@ +<section id='package-ebuild-eapi-4-slot-abi'> +<title>EAPI 4-slot-abi</title> +<section id='package-ebuild-eapi-4-slot-abi-metadata'> +<title>Metadata</title> +<section id='package-ebuild-eapi-4-slot-abi-metadata-slot-sub-slot-abi'> +<title>SLOT Supports Optional "sub-slot" ABI part</title> +<para> +In order to represent cases in which an upgrade to a new version of a package +requires reverse dependencies to be rebuilt, the SLOT variable may contain an +optional "sub-slot" ABI part that is delimited by a '/' character. +</para> +<para> +For +example, the package 'dev-libs/glib-2.30.2' may set SLOT="2/2.30" in order to +indicate a sub-slot value of "2.30". This package will be matched by +dependency atoms such as 'dev-libs/glib:2' or 'dev-libs/glib:2/2.30', where +the sub-slot part of the atom is optional. +</para> +<para> +If SLOT does not contain a sub-slot +part, then it is considered to have an implicit sub-slot that is equal to the +SLOT value. For example, SLOT="0" is implicitly equal to SLOT="0/0". +</para> +<para> +Refer to the +<link linkend="package-ebuild-eapi-4-slot-abi-metadata-dependency-atom-slot-abi-equal-operator"> +:= operator </link> documentation for more information about sub-slot usage. +</para> +</section> +<section id='package-ebuild-eapi-4-slot-abi-metadata-dependency-atom-slot-abi-equal-operator'> +<title>Dependency Atom SLOT/ABI := Operator</title> +<para> +Dependency atom syntax now supports SLOT/ABI := operators which allow the +specific SLOT/ABI that a package is built against to be recorded, so that it's +possible to automatically determine when a package needs to be rebuilt due to +having a dependency upgraded to a different SLOT/ABI. +</para> +<para> +For example, if a package is built +against the package 'dev-libs/glib-2.30.2' with SLOT="2/2.30", then dependency +atoms such as 'dev-libs/glib:=' or 'dev-libs/glib:2=' will be rewritten at +build time to be recorded as 'dev-libs/glib:2/2.30='. +</para> +<para> +For another example, if +a package is built against the package 'sys-libs/db-4.8.30' with SLOT="4.8", +then a dependency atom such as 'sys-libs/db:=' will be rewritten at build time +to be recorded as 'sys-libs/db:4.8/4.8='. In this case, since SLOT="4.8" does +not contain a sub-slot part, the sub-slot is considered to be implicitly equal +to "4.8". +</para> +<para> +When dependencies are rewritten as described above, the SLOT/ABI recorded in +the atom is always equal to that of the highest matched version that is +installed at build time. +</para> +</section> +<section id='package-ebuild-eapi-4-slot-abi-metadata-dependency-atom-slot-abi-asterisk-operator'> +<title>Dependency Atom SLOT/ABI :* Operator</title> +<para> +The new :* operator is used to express dependencies that can change versions +at runtime without requiring reverse dependencies to be rebuilt. For example, +a dependency atom such as 'dev-libs/glib:*' can be used to match any slot of +the 'dev-libs/glib' package, and dependency atom such as 'dev-libs/glib:2*' +can be used to specifically match slot '2' of the same package (ignoring its +sub-slot). +</para> +</section> +</section> +</section> diff --git a/doc/package/ebuild/phases.docbook b/doc/package/ebuild/phases.docbook index f3e8359ef83..8bfaeeaf75a 100644 --- a/doc/package/ebuild/phases.docbook +++ b/doc/package/ebuild/phases.docbook @@ -13,12 +13,21 @@ </para> <itemizedlist> <listitem> + <para>pkg_pretend</para> + </listitem> + <listitem> <para>pkg_setup</para> </listitem> <listitem> <para>src_unpack</para> </listitem> <listitem> + <para>src_prepare</para> + </listitem> + <listitem> + <para>src_configure</para> + </listitem> + <listitem> <para>src_compile</para> </listitem> <listitem> diff --git a/doc/portage.docbook b/doc/portage.docbook index c0121b8d08b..781915cbb55 100644 --- a/doc/portage.docbook +++ b/doc/portage.docbook @@ -21,6 +21,7 @@ <!ENTITY package_ebuild_eapi_3 SYSTEM "package/ebuild/eapi/3.docbook"> <!ENTITY package_ebuild_eapi_4 SYSTEM "package/ebuild/eapi/4.docbook"> <!ENTITY package_ebuild_eapi_4_python SYSTEM "package/ebuild/eapi/4-python.docbook"> + <!ENTITY package_ebuild_eapi_4_slot_abi SYSTEM "package/ebuild/eapi/4-slot-abi.docbook"> <!ENTITY qa SYSTEM "qa.docbook"> <!ENTITY config SYSTEM "config.docbook"> <!ENTITY config_bashrc SYSTEM "config/bashrc.docbook"> diff --git a/man/ebuild.5 b/man/ebuild.5 index 42b72c88d3c..e9e718e0d7f 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -202,7 +202,7 @@ who uses the \fBebuild\fR(1) and \fBrepoman\fR(1) commands with this package will be required to have a version of portage that recognizes the EAPI to which this package conforms. .TP -\fBSRC_URI\fR = \fI"http://happy.com/little/${P}.tar.gz"\fR +\fBSRC_URI\fR = \fI"http://example.com/path/${P}.tar.gz"\fR Contains a list of URIs for the required source files. It can contain multiple URIs for a single source file. The list is processed in order if the file was not found on any of the \fIGENTOO_MIRRORS\fR. @@ -211,7 +211,7 @@ customized with a "->" operator on the right hand side, followed by the desired output file name. All tokens, including the operator and output file name, should be separated by whitespace. .TP -\fBHOMEPAGE\fR = \fI"http://happy.com/"\fR +\fBHOMEPAGE\fR = \fI"http://example.com/"\fR Should contain a list of URIs for the sources main sites and other further package dependent information. .TP @@ -394,7 +394,7 @@ immediately following a flag with either \fB(+)\fR or \fB(\-)\fR. Use .RS .nf media\-video/ffmpeg[threads(+)] -media\-video/ffmpeg[threads(\-)] +media\-video/ffmpeg[-threads(\-)] .fi .RE .RE @@ -468,9 +468,9 @@ will use that to consider the dependency satisfied. .TP \fBRDEPEND\fR This should contain a list of all packages that are required for this -program to run (aka runtime depend). If this is not set in \fBEAPI\fR -3 or earlier, then it defaults to the value of \fBDEPEND\fR. In -EAPI 4 or later, \fBRDEPEND\fR will never be implicitly set. +program to run (aka runtime depend). If this is not set in \fBEAPI 3\fR +or earlier, then it defaults to the value of \fBDEPEND\fR. In +\fBEAPI 4\fR or later, \fBRDEPEND\fR will never be implicitly set. .br You may use the same syntax to vary dependencies as seen above in \fBDEPEND\fR. .TP @@ -615,7 +615,7 @@ CFLAGS, CXXFLAGS, FFLAGS, FCFLAGS, and LDFLAGS variables. This should contain a list of file paths, relative to the image directory, of files that contain .hash sections. The paths may contain regular expressions with escape\-quoted special characters. This variable is deprecated. Use -\fBQA_FLAGS_IGNORED\f instead. +\fBQA_FLAGS_IGNORED\fR instead. .br This variable is intended to be used on files of binary packages which ignore LDFLAGS variable. @@ -1105,8 +1105,8 @@ that this expression does \fBNOT\fR use the offset prefix. .BR 'dosed\ "s:/usr/local:/usr:g"\ /usr/bin/some\-script' runs sed on ${ED}/usr/bin/some\-script .TP -\fBdodir\fR \fI<path>\fR -Creates a directory inside of ${ED}. +\fBdodir\fR \fI<path> [more paths]\fR +Creates directories inside of ${ED}. .br .BR 'dodir\ /usr/lib/apache' creates ${ED}/usr/lib/apache. Note that the do* functions will run @@ -1122,8 +1122,8 @@ Sets the root (\fIDESTTREE\fR) for other functions like \fBdobin\fR, .br The default root is /usr. .TP -\fBkeepdir\fR \fI<path>\fR -Tells portage to leave a directory behind even if it is empty. Functions +\fBkeepdir\fR \fI<path> [more paths]\fR +Tells portage to leave directories behind even if they're empty. Functions the same as \fBdodir\fR. .TP \fBdobin\fR \fI<binary> [list of more binaries]\fR diff --git a/man/emaint.1 b/man/emaint.1 index dff6fddd90b..c588a0bfe59 100644 --- a/man/emaint.1 +++ b/man/emaint.1 @@ -19,9 +19,17 @@ Generate a metadata index for binary packages located in \fBPKGDIR\fR (for download by remote clients). See the \fBPORTAGE_BINHOST\fR documentation in the \fBmake.conf\fR(5) man page for additional information. .TP +.BR cleanconfig +Discard no longer installed config tracker entries. +.TP .BR cleanresume Discard merge lists saved for the \fBemerge\fR(1) \fB--resume\fR action. .TP +.BR logs +Clean out old logs from the \fBPORT_LOGDIR\fR using the command \fBPORT_LOGDIR_CLEAN\fR +See the \fBmake.conf\fR(5) man page for additional information as well as enabling the +\fB'clean-logs'\fR feature in emerge to do this automatically. +.TP .BR movebin Perform package move updates for binary packages located in \fBPKGDIR\fR. .TP @@ -30,23 +38,37 @@ Perform package move updates for installed packages. .TP .BR world Fix problems in the \fIworld\fR file. -.SH OPTIONS +.SH DEFAULT OPTIONS .TP .B \-c, \-\-check -Check for any problems that may exist. +Check for any problems that may exist. (all commands) .TP .B \-f, \-\-fix -Fix any problems that may exist. +Fix any problems that may exist. (not all commands) +.SH OPTIONS +.TP +.B \-C, \-\-clean +Cleans the logs from \fBPORT_LOGDIR\fR (logs command only) +.TP +.B \-p, \-\-pretend +Sets pretend mode (same as \-c, \-\-check) for use with the \-C, \-\-clean OPTION (logs command only) +.TP +.B \-t NUM, \-\-time NUM +Changes the minimum age \fBNUM\fR (in days) of the logs to be listed or deleted. (logs command only) .SH "REPORTING BUGS" Please report bugs via http://bugs.gentoo.org/ .SH AUTHORS .nf Mike Frysinger <vapier@gentoo.org> +Brian Dolbec <dolsen@gentoo.org> .fi .SH "FILES" .TP .B /var/lib/portage/world Contains a list of all user\-specified packages. +.TP +.B /var/lib/portage/config +Contains the paths and md5sums of all the config files being tracked. .SH "SEE ALSO" .BR emerge (1), .BR portage (5) diff --git a/man/emerge.1 b/man/emerge.1 index 7b59040cbf7..7aa46226cbc 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -1,4 +1,4 @@ -.TH "EMERGE" "1" "Dec 2011" "Portage VERSION" "Portage" +.TH "EMERGE" "1" "Jun 2012" "Portage VERSION" "Portage" .SH "NAME" emerge \- Command\-line interface to the Portage system .SH "SYNOPSIS" @@ -69,7 +69,9 @@ packages deemed necessary for your system to run properly. \fBworld\fR encompasses both the \fBselected\fR and \fBsystem\fR sets. [See \fBFILES\fR below for more information.] Other sets can exist depending on the current configuration. The default set configuration is located -in the \fB/usr/share/portage/config/sets\fR directory. Note that a \fIset\fR +in the \fB/usr/share/portage/config/sets\fR directory. +User sets may be created by placing files in the \fB/etc/portage/sets/\fR +directory (see \fBportage\fR(5)). Note that a \fIset\fR is generally used in conjunction with \fB\-\-update\fR. When used as arguments to \fBemerge\fR sets have to be prefixed with \fB@\fR to be recognized. Use the \fB\-\-list\-sets\fR action to display a list of @@ -200,6 +202,9 @@ generate and distribute cache for use by others, use \fBegencache\fR(1). .TP .BR "\-\-resume" (\fB\-r\fR) Resumes the most recent merge list that has been aborted due to an error. +This re\-uses the options that were given with the original +command that's being resumed, and the user may also provide +additional options when calling \fB\-\-resume\fR. Please note that this operation will only return an error on failure. If there is nothing for portage to do, then portage will exit with a message and a success condition. A resume list will persist until it has been completed in @@ -391,6 +396,10 @@ be updated than would have otherwise been updated with the option disabled. Using \fB\-\-with\-bdeps=y\fR together with \fB\-\-complete\-graph\fR makes the graph as complete as possible. .TP +.BR "\-\-complete\-graph\-if\-new\-use < y | n >" +Trigger the \fB\-\-complete\-graph\fR behavior if USE or IUSE will +change for an installed package. This option is enabled by default. +.TP .BR "\-\-complete\-graph\-if\-new\-ver < y | n >" Trigger the \fB\-\-complete\-graph\fR behavior if an installed package version will change (upgrade or downgrade). This option is enabled by default. @@ -476,6 +485,13 @@ remote server are preferred over local packages if they are not identical. .BR "\-\-ignore-default-opts" Causes \fIEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) to be ignored. .TP +.BR "\-\-ignore\-built\-slot\-abi\-deps < y | n >" +Ignore the SLOT/ABI := operator parts of dependencies that have +been recorded when packages where built. This option is intended +only for debugging purposes, and it only affects built packages +that specify SLOT/ABI := operator dependencies using the +experimental "4\-slot\-abi" EAPI. +.TP .BR "-j [JOBS], \-\-jobs[=JOBS]" Specifies the number of packages to build simultaneously. If this option is given without an argument, emerge will not limit the number of jobs that can @@ -536,7 +552,7 @@ the build may fail if the dependencies aren't satisfied. .TP .BR "\-\-noreplace " (\fB\-n\fR) Skips the packages specified on the command\-line that have already -been installed. Without this option, any packages, ebuilds, or deps +been installed. Without this option, any package atoms or package sets you specify on the command\-line \fBwill\fR cause Portage to remerge the package, even if it is already installed. Note that Portage will not remerge dependencies by default. @@ -629,6 +645,17 @@ Disable the warning message that's shown prior to to be set in the \fBmake.conf\fR(5) \fBEMERGE_DEFAULT_OPTS\fR variable. .TP +.BR "\-\-rebuild\-if\-new\-slot\-abi [ y | n ]" +Automatically rebuild or reinstall packages when SLOT/ABI := +operator dependencies can be satisfied by a newer slot, so that +older packages slots will become eligible for removal by the +\-\-depclean action as soon as possible. This option only +affects packages that specify SLOT/ABI dependencies using the +experimental "4\-slot\-abi" EAPI. Since this option requires +checking of reverse dependencies, it enables \-\-complete\-graph +mode whenever a new slot is installed. This option is enabled by +default. +.TP .BR "\-\-rebuild\-if\-new\-rev [ y | n ]" Rebuild packages when build\-time dependencies are built from source, if the dependency is not already installed with the same version and revision. @@ -897,7 +924,7 @@ those USE settings (on Bourne-compatible shells you may omit the \fBenv\fR part). If you want those USE settings to be more permanent, you can put them in /etc/portage/package.use instead. .LP -If \fBemerge \-\-update system\fR or \fBemerge \-\-update world\fR +If \fBemerge \-\-update @system\fR or \fBemerge \-\-update @world\fR fails with an error message, it may be that an ebuild uses some newer feature not present in this version of \fBemerge\fR. You can use \fBemerge \-\-update portage\fR to upgrade to the lastest @@ -1036,6 +1063,11 @@ Contains a list of all user\-specified packages. You can safely edit this file, adding packages that you want to be considered in \fBworld\fR set updates and removing those that you do not want to be considered. .TP +.B /var/lib/portage/world_sets +This is like the world file but instead of package atoms it contains +packages sets which always begin with the \fB@\fR character. Use +\fB/etc/portage/sets/\fR to define user package sets. +.TP .B /etc/make.conf Contains variables for the build process, overriding those in \fBmake.globals\fR. @@ -1043,6 +1075,9 @@ Contains variables for the build process, overriding those in .B /etc/portage/color.map Contains variables customizing colors. .TP +.B /etc/portage/sets/ +Contains user package set definitions (see \fBportage\fR(5)). +.TP .B /etc/dispatch\-conf.conf Contains settings to handle automatic updates/backups of configuration files. diff --git a/man/make.conf.5 b/man/make.conf.5 index e8777c84050..876a8a330fe 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -1,4 +1,4 @@ -.TH "MAKE.CONF" "5" "Dec 2011" "Portage VERSION" "Portage" +.TH "MAKE.CONF" "5" "Jul 2012" "Portage VERSION" "Portage" .SH "NAME" make.conf \- custom settings for Portage .SH "SYNOPSIS" @@ -128,11 +128,13 @@ Determines how long the countdown delay will be after running .br Defaults to 5 seconds. .TP -\fBCOLLISION_IGNORE\fR = \fI[space delimited list of files and/or directories]\fR +\fBCOLLISION_IGNORE\fR = \fI[space delimited list of fnmatch patterns]\fR This variable allows the user to disable \fIcollision\-protect\fR and -\fIprotect\-owned\fR for specific files and/or directories. +\fIprotect\-owned\fR for specific \fBfnmatch\fR(3) patterns. For backward +compatibility, directories that are listed without a fnmatch pattern will +automatically have /* appended to them. .br -Defaults to /lib/modules. +Defaults to "/lib/modules/* *.py[co]". .TP \fBCONFIG_PROTECT\fR = \fI[space delimited list of files and/or directories]\fR All files and/or directories that are defined here will have "config file protection" @@ -268,7 +270,8 @@ information (upon which this feature depends). .TP .B config\-protect\-if\-modified This causes the \fBCONFIG_PROTECT\fR behavior to be skipped for files -that have not been modified since they were installed. +that have not been modified since they were installed. This feature is +enabled by default. .TP .B digest Autogenerate digests for packages when running the @@ -290,6 +293,12 @@ strangely configured Samba server (oplocks off, NFS re\-export). A tool /usr/lib/portage/bin/clean_locks exists to help handle lock issues when a problem arises (normally due to a crash or disconnect). .TP +.B downgrade\-backup +When a package is downgraded to a lower version, call \fBquickpkg\fR(1) +in order to create a backup of the installed version before it is +unmerged (if a binary package of the same version does not already +exist). Also see the related \fIunmerge\-backup\fR feature. +.TP .B ebuild\-locks Use locks to ensure that unsandboxed ebuild phases never execute concurrently. Also see \fIparallel\-install\fR. @@ -405,9 +414,9 @@ parallelization. For additional parallelization, disable \fIebuild\-locks\fR. .TP .B parse\-eapi\-ebuild\-head -Parse \fBEAPI\fR from the head of the ebuild (first 30 lines). This feature -is only intended for experimental purposes and should not be enabled under -normal circumstances. +Parse \fBEAPI\fR from the head of the ebuild as specified in PMS section +7.3.1, and treat non\-conformant ebuilds as invalid. This feature is +enabled by default, and will soon become enabled unconditionally. .TP .B prelink\-checksums If \fBprelink\fR(8) is installed then use it to undo any prelinks on files @@ -511,6 +520,11 @@ continue to execute the remaining phases as if the failure had not occurred. Note that the test phase for a specific package may be disabled by masking the "test" \fBUSE\fR flag in \fBpackage.use.mask\fR (see \fBportage\fR(5)). .TP +.B unmerge\-backup +Call \fBquickpkg\fR(1) to create a backup of each package before it is +unmerged (if a binary package of the same version does not already exist). +Also see the related \fIdowngrade\-backup\fR feature. +.TP .B unmerge\-logs Keep logs from successful unmerge phases. This is relevant only when \fBPORT_LOGDIR\fR is set. @@ -730,6 +744,24 @@ to be configured by \fI`git config user.signingkey key_id`\fR. The command used by \fBrepoman\fR(1) to sign manifests when \fBsign\fR is in \fBFEATURES\fR. .TP +\fBPORTAGE_GRPNAME\fR = \fI[group]\fR +Defines the groupname to use when executing in userpriv/etc... modes (i.e. +non-root). +.br +Defaults to portage. +.TP +\fBPORTAGE_INST_GID\fR = \fI[gid]\fR +Defines the group id when installing files via dobin/dosbin. Useful when +running ebuild as yourself. +.br +Defaults to 0. +.TP +\fBPORTAGE_INST_UID\fR = \fI[uid]\fR +Defines the user id when installing files via dobin/dosbin. Useful when +running ebuild as yourself. +.br +Defaults to 0. +.TP \fBPORTAGE_IONICE_COMMAND\fR = \fI[ionice command string]\fR This variable should contain a command for portage to call in order to adjust the io priority of portage and it's subprocesses. The command @@ -789,6 +821,12 @@ Defines the location of the temporary build directories. .br Defaults to /var/tmp. .TP +\fBPORTAGE_USERNAME\fR = \fI[user]\fR +Defines the username to use when executing in userpriv/etc... modes (i.e. +non-root). +.br +Defaults to portage. +.TP \fBPORTAGE_WORKDIR_MODE\fR = \fI"0700"\fR This variable controls permissions for \fIWORKDIR\fR (see \fBebuild\fR(5)). .TP @@ -869,6 +907,14 @@ is used to sync the local portage tree when `emerge \-\-sync` is run. .br Defaults to rsync://rsync.gentoo.org/gentoo\-portage .TP +\fBUNINSTALL_IGNORE\fR = \fI[space delimited list of fnmatch patterns]\fR +This variable prevents uninstallation of files that match +specific \fBfnmatch\fR(3) patterns. In order to ignore file +collisions with these files at install time, the same patterns +can be added to the \fBCOLLISION_IGNORE\fR variable. +.br +Defaults to "/lib/modules/*". +.TP \fBUSE\fR = \fI[space delimited list of USE items]\fR This variable contains options that control the build behavior of several packages. More information in \fBebuild\fR(5). Possible USE values diff --git a/man/portage.5 b/man/portage.5 index 099d6011d5f..ad84ff1a758 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1,4 +1,4 @@ -.TH "PORTAGE" "5" "Oct 2011" "Portage VERSION" "Portage" +.TH "PORTAGE" "5" "Jun 2012" "Portage VERSION" "Portage" .SH NAME portage \- the heart of Gentoo .SH "DESCRIPTION" @@ -69,6 +69,9 @@ package-specific bashrc files .BR /etc/portage/profile/ site-specific overrides of \fB/etc/make.profile/\fR .TP +.BR /etc/portage/sets/ +user\-defined package sets +.TP .BR /usr/portage/metadata/ .nf layout.conf @@ -159,6 +162,9 @@ next to each other. .I Examples: .nf +# match anything with a version containing 9999, which can be used in +# package.mask to prevent emerge --autounmask from selecting live ebuilds +=*/*-*9999* # match anything from the 'sys\-apps' category sys\-apps/* # match packages named 'zlib' from any category @@ -761,6 +767,17 @@ in the following order: /etc/portage/env/${CATEGORY}/${PF} .RE .TP +.BR /etc/portage/sets/ +.RS +For each file in this directory, a package set is created with its name +corresponding to the name of the file. Each file should contain a list +of package atoms, one per line. When referencing package sets in +\fBemerge\fR(1) arguments, the set name is prefixed with \fB@\fR. + +Also see \fB/var/lib/portage/world_sets\fR and the \fBemerge\fR(1) +\fB\-\-list\-sets\fR option. +.RE +.TP .BR /usr/portage/metadata/ .RS .TP @@ -808,7 +825,12 @@ cache\-formats = md5-dict pms # indicate that this repo contains profiles that may use directories for # package.mask, package.provided, package.use, package.use.mask, # package.use.force, use.mask and use.force. -profile\-formats = portage-1 +# profile\-formats = portage-1 +# indicate that paths such as 'gentoo:targets/desktop' or ':targets/desktop' in +# profile parent files can be used to express paths relative to the root +# 'profiles' directory of a repository (when the repo name is omitted before +# the colon, it refers to the current repository the parent file is inside) +profile\-formats = portage-2 .fi .RE .TP @@ -1077,7 +1099,8 @@ app\-cdr/cdemu .TP .BR world_sets This is like the world file but instead of package atoms it contains -packages sets which always begin with the @ character. +packages sets which always begin with the \fB@\fR character. Use +\fB/etc/portage/sets/\fR to define user package sets. .I Example: .nf diff --git a/man/repoman.1 b/man/repoman.1 index 37babcd4bf7..b8c0f48e39f 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -1,4 +1,4 @@ -.TH "REPOMAN" "1" "Mar 2012" "Portage VERSION" "Portage" +.TH "REPOMAN" "1" "June 2012" "Portage VERSION" "Portage" .SH NAME repoman \- Gentoo's program to enforce a minimal level of quality assurance in packages added to the portage tree .SH SYNOPSIS @@ -146,7 +146,8 @@ Syntax error in DEPEND (usually an extra/missing space/parenthesis) Ebuilds that have a missing or empty DESCRIPTION variable .TP .B EAPI.definition -EAPI is defined after an inherit call (must be defined before) +EAPI definition does not conform to PMS section 7.3.1 (first +non\-comment, non\-blank line). See bug #402167. .TP .B EAPI.deprecated Ebuilds that use features that are deprecated in the current EAPI @@ -320,9 +321,6 @@ PATCHES variable should be a bash array to ensure white space safety Error generating cache entry for ebuild; typically caused by ebuild syntax error or digest verification failure. .TP -.B eprefixify.defined -The ebuild uses eprefixify, but does not inherit the prefix eclass -.TP .B file.UTF8 File is not UTF8 compliant .TP @@ -335,8 +333,11 @@ File/dir name must be composed of only the following chars: a-zA-Z0-9._-+: .B file.size Files in the files directory must be under 20k .TP -.B inherit.autotools -Ebuild inherits autotools but does not call eautomake, eautoconf or eautoreconf +.B inherit.missing +Ebuild uses functions from an eclass but does not inherit it +.TP +.B inherit.unused +Ebuild inherits an eclass but does not use it .TP .B inherit.deprecated Ebuild inherits a deprecated eclass @@ -381,6 +382,10 @@ Ebuild uses D, ROOT, ED, EROOT or EPREFIX with helpers The ebuild PROVIDEs an old-style virtual (see GLEP 37). This is an error unless "allow\-provide\-virtuals = true" is set in metadata/layout.conf. .TP +.B virtual.suspect +Ebuild contains a package that usually should be pulled via virtual/, +not directly. +.TP .B wxwidgets.eclassnotused Ebuild DEPENDs on x11-libs/wxGTK without inheriting wxwidgets.eclass. Refer to bug #305469 for more information. diff --git a/pym/_emerge/AtomArg.py b/pym/_emerge/AtomArg.py index a929b433e9a..343d7aaabd5 100644 --- a/pym/_emerge/AtomArg.py +++ b/pym/_emerge/AtomArg.py @@ -1,10 +1,13 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage._sets.base import InternalPackageSet from _emerge.DependencyArg import DependencyArg class AtomArg(DependencyArg): + + __slots__ = ('atom', 'pset') + def __init__(self, atom=None, **kwargs): DependencyArg.__init__(self, **kwargs) self.atom = atom diff --git a/pym/_emerge/BlockerCache.py b/pym/_emerge/BlockerCache.py index 06598a44ec1..fce81f83abf 100644 --- a/pym/_emerge/BlockerCache.py +++ b/pym/_emerge/BlockerCache.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -16,6 +16,9 @@ except ImportError: if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class BlockerCache(portage.cache.mappings.MutableMapping): """This caches blockers of installed packages so that dep_check does not @@ -159,8 +162,8 @@ class BlockerCache(portage.cache.mappings.MutableMapping): @param blocker_data: An object with counter and atoms attributes. @type blocker_data: BlockerData """ - self._cache_data["blockers"][cpv] = \ - (blocker_data.counter, tuple(str(x) for x in blocker_data.atoms)) + self._cache_data["blockers"][_unicode(cpv)] = (blocker_data.counter, + tuple(_unicode(x) for x in blocker_data.atoms)) self._modified.add(cpv) def __iter__(self): diff --git a/pym/_emerge/Dependency.py b/pym/_emerge/Dependency.py index c2d36b2dcaa..2ec860f8363 100644 --- a/pym/_emerge/Dependency.py +++ b/pym/_emerge/Dependency.py @@ -6,7 +6,7 @@ from _emerge.DepPriority import DepPriority class Dependency(SlotObject): __slots__ = ("atom", "blocker", "child", "depth", - "parent", "onlydeps", "priority", "root", + "parent", "onlydeps", "priority", "root", "want_update", "collapsed_parent", "collapsed_priority") def __init__(self, **kwargs): SlotObject.__init__(self, **kwargs) diff --git a/pym/_emerge/DependencyArg.py b/pym/_emerge/DependencyArg.py index 861d837546d..80134c804f7 100644 --- a/pym/_emerge/DependencyArg.py +++ b/pym/_emerge/DependencyArg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -6,8 +6,19 @@ import sys from portage import _encodings, _unicode_encode, _unicode_decode class DependencyArg(object): - def __init__(self, arg=None, root_config=None): + + __slots__ = ('arg', 'force_reinstall', 'internal', 'reset_depth', 'root_config') + + def __init__(self, arg=None, force_reinstall=False, internal=False, + reset_depth=True, root_config=None): + """ + Use reset_depth=False for special arguments that should not interact + with depth calculations (see the emerge --deep=DEPTH option). + """ self.arg = arg + self.force_reinstall = force_reinstall + self.internal = internal + self.reset_depth = reset_depth self.root_config = root_config def __eq__(self, other): diff --git a/pym/_emerge/EbuildBinpkg.py b/pym/_emerge/EbuildBinpkg.py index b7d43ba82ac..34a6aef9c6b 100644 --- a/pym/_emerge/EbuildBinpkg.py +++ b/pym/_emerge/EbuildBinpkg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.CompositeTask import CompositeTask @@ -34,6 +34,10 @@ class EbuildBinpkg(CompositeTask): self.settings.pop("PORTAGE_BINPKG_TMPFILE", None) if self._default_exit(package_phase) != os.EX_OK: + try: + os.unlink(self._binpkg_tmpfile) + except OSError: + pass self.wait() return diff --git a/pym/_emerge/EbuildMetadataPhase.py b/pym/_emerge/EbuildMetadataPhase.py index a34542d4374..c2d3747f7f4 100644 --- a/pym/_emerge/EbuildMetadataPhase.py +++ b/pym/_emerge/EbuildMetadataPhase.py @@ -5,10 +5,14 @@ from _emerge.SubProcess import SubProcess import sys from portage.cache.mappings import slot_dict_class import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.package.ebuild._eapi_invalid:eapi_invalid', +) from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode + import errno import fcntl import io @@ -20,39 +24,44 @@ class EbuildMetadataPhase(SubProcess): used to extract metadata from the ebuild. """ - __slots__ = ("cpv", "eapi", "ebuild_hash", "fd_pipes", "metadata_callback", + __slots__ = ("cpv", "eapi_supported", "ebuild_hash", "fd_pipes", "metadata", "portdb", "repo_path", "settings") + \ - ("_raw_metadata",) + ("_eapi", "_eapi_lineno", "_raw_metadata",) _file_names = ("ebuild",) _files_dict = slot_dict_class(_file_names, prefix="") _metadata_fd = 9 def _start(self): - settings = self.settings - settings.setcpv(self.cpv) ebuild_path = self.ebuild_hash.location - # the caller can pass in eapi in order to avoid - # redundant _parse_eapi_ebuild_head calls - eapi = self.eapi - if eapi is None and \ - 'parse-eapi-ebuild-head' in settings.features: - with io.open(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - eapi = portage._parse_eapi_ebuild_head(f) - - if eapi is not None: - if not portage.eapi_is_supported(eapi): - self.metadata = self.metadata_callback(self.cpv, - self.repo_path, {'EAPI' : eapi}, self.ebuild_hash) - self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() - return - - settings.configdict['pkg']['EAPI'] = eapi + with io.open(_unicode_encode(ebuild_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + self._eapi, self._eapi_lineno = portage._parse_eapi_ebuild_head(f) + + parsed_eapi = self._eapi + if parsed_eapi is None: + parsed_eapi = "0" + + if not parsed_eapi: + # An empty EAPI setting is invalid. + self._eapi_invalid(None) + self._set_returncode((self.pid, 1 << 8)) + self.wait() + return + + self.eapi_supported = portage.eapi_is_supported(parsed_eapi) + if not self.eapi_supported: + self.metadata = {"EAPI": parsed_eapi} + self._set_returncode((self.pid, os.EX_OK << 8)) + self.wait() + return + + settings = self.settings + settings.setcpv(self.cpv) + settings.configdict['pkg']['EAPI'] = parsed_eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None @@ -139,12 +148,52 @@ class EbuildMetadataPhase(SubProcess): metadata_lines = _unicode_decode(b''.join(self._raw_metadata), encoding=_encodings['repo.content'], errors='replace').splitlines() + metadata_valid = True if len(portage.auxdbkeys) != len(metadata_lines): # Don't trust bash's returncode if the # number of lines is incorrect. - self.returncode = 1 + metadata_valid = False else: - metadata = zip(portage.auxdbkeys, metadata_lines) - self.metadata = self.metadata_callback(self.cpv, - self.repo_path, metadata, self.ebuild_hash) + metadata = dict(zip(portage.auxdbkeys, metadata_lines)) + parsed_eapi = self._eapi + if parsed_eapi is None: + parsed_eapi = "0" + self.eapi_supported = \ + portage.eapi_is_supported(metadata["EAPI"]) + if (not metadata["EAPI"] or self.eapi_supported) and \ + metadata["EAPI"] != parsed_eapi: + self._eapi_invalid(metadata) + if 'parse-eapi-ebuild-head' in self.settings.features: + metadata_valid = False + + if metadata_valid: + # Since we're supposed to be able to efficiently obtain the + # EAPI from _parse_eapi_ebuild_head, we don't write cache + # entries for unsupported EAPIs. + if self.eapi_supported: + + if metadata.get("INHERITED", False): + metadata["_eclasses_"] = \ + self.portdb.repositories.get_repo_for_location( + self.repo_path).eclass_db.get_eclass_data( + metadata["INHERITED"].split()) + else: + metadata["_eclasses_"] = {} + metadata.pop("INHERITED", None) + + self.portdb._write_cache(self.cpv, + self.repo_path, metadata, self.ebuild_hash) + else: + metadata = {"EAPI": metadata["EAPI"]} + self.metadata = metadata + else: + self.returncode = 1 + def _eapi_invalid(self, metadata): + repo_name = self.portdb.getRepositoryName(self.repo_path) + if metadata is not None: + eapi_var = metadata["EAPI"] + else: + eapi_var = None + eapi_invalid(self, self.cpv, repo_name, self.settings, + eapi_var, self._eapi, self._eapi_lineno) diff --git a/pym/_emerge/EbuildPhase.py b/pym/_emerge/EbuildPhase.py index 27944f413d5..fe44abcbd27 100644 --- a/pym/_emerge/EbuildPhase.py +++ b/pym/_emerge/EbuildPhase.py @@ -12,15 +12,24 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess from _emerge.EbuildProcess import EbuildProcess from _emerge.CompositeTask import CompositeTask from portage.util import writemsg -from portage.xml.metadata import MetaDataXML + +try: + from portage.xml.metadata import MetaDataXML +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 + MetaDataXML = None + import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.elog:messages@elog_messages', 'portage.package.ebuild.doebuild:_check_build_log,' + \ '_post_phase_cmds,_post_phase_userpriv_perms,' + \ - '_post_src_install_chost_fix,' + \ '_post_src_install_soname_symlinks,' + \ '_post_src_install_uid_fix,_postinst_bsdflags,' + \ + '_post_src_install_write_metadata,' + \ '_preinst_bsdflags' ) from portage import os @@ -71,7 +80,7 @@ class EbuildPhase(CompositeTask): maint_str = "" upstr_str = "" metadata_xml_path = os.path.join(os.path.dirname(self.settings['EBUILD']), "metadata.xml") - if os.path.isfile(metadata_xml_path): + if MetaDataXML is not None and os.path.isfile(metadata_xml_path): herds_path = os.path.join(self.settings['PORTDIR'], 'metadata/herds.xml') try: @@ -206,7 +215,7 @@ class EbuildPhase(CompositeTask): if self.phase == "install": out = io.StringIO() - _post_src_install_chost_fix(settings) + _post_src_install_write_metadata(settings) _post_src_install_uid_fix(settings, out) msg = out.getvalue() if msg: diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py index d4dbe97ec47..ce15f5a36fe 100644 --- a/pym/_emerge/FakeVartree.py +++ b/pym/_emerge/FakeVartree.py @@ -10,11 +10,17 @@ from _emerge.Package import Package from _emerge.PackageVirtualDbapi import PackageVirtualDbapi from portage.const import VDB_PATH from portage.dbapi.vartree import vartree +from portage.dep._slot_abi import find_built_slot_abi_atoms +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidDependString from portage.repository.config import _gen_valid_repo from portage.update import grab_updates, parse_updates, update_dbentries if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode class FakeVardbapi(PackageVirtualDbapi): """ @@ -39,9 +45,10 @@ class FakeVartree(vartree): is not a matching ebuild in the tree). Instances of this class are not populated until the sync() method is called.""" def __init__(self, root_config, pkg_cache=None, pkg_root_config=None, - dynamic_deps=True): + dynamic_deps=True, ignore_built_slot_abi_deps=False): self._root_config = root_config self._dynamic_deps = dynamic_deps + self._ignore_built_slot_abi_deps = ignore_built_slot_abi_deps if pkg_root_config is None: pkg_root_config = self._root_config self._pkg_root_config = pkg_root_config @@ -68,7 +75,7 @@ class FakeVartree(vartree): self.dbapi.aux_get = self._aux_get_wrapper self.dbapi.match = self._match_wrapper self._aux_get_history = set() - self._portdb_keys = ["EAPI", "DEPEND", "RDEPEND", "PDEPEND"] + self._portdb_keys = ["EAPI", "KEYWORDS", "DEPEND", "RDEPEND", "PDEPEND"] self._portdb = portdb self._global_updates = None @@ -101,7 +108,18 @@ class FakeVartree(vartree): self._aux_get_history.add(pkg) # We need to check the EAPI, and this also raises # a KeyError to the caller if appropriate. - installed_eapi, repo = self._aux_get(pkg, ["EAPI", "repository"]) + pkg_obj = self.dbapi._cpv_map[pkg] + installed_eapi = pkg_obj.metadata['EAPI'] + repo = pkg_obj.metadata['repository'] + eapi_attrs = _get_eapi_attrs(installed_eapi) + built_slot_abi_atoms = None + + if eapi_attrs.slot_abi and not self._ignore_built_slot_abi_deps: + try: + built_slot_abi_atoms = find_built_slot_abi_atoms(pkg_obj) + except InvalidDependString: + pass + try: # Use the live ebuild metadata if possible. repo = _gen_valid_repo(repo) @@ -118,6 +136,16 @@ class FakeVartree(vartree): if not (portage.eapi_is_supported(live_metadata["EAPI"]) and \ portage.eapi_is_supported(installed_eapi)): raise KeyError(pkg) + + # preserve built SLOT/ABI := operator deps + if built_slot_abi_atoms: + live_eapi_attrs = _get_eapi_attrs(live_metadata["EAPI"]) + if not live_eapi_attrs.slot_abi: + raise KeyError(pkg) + for k, v in built_slot_abi_atoms.items(): + live_metadata[k] += (" " + + " ".join(_unicode(atom) for atom in v)) + self.dbapi.aux_update(pkg, live_metadata) except (KeyError, portage.exception.PortageException): if self._global_updates is None: @@ -258,8 +286,9 @@ def grab_global_updates(portdb): return retupdates def perform_global_updates(mycpv, mydb, myupdates): - aux_keys = ["DEPEND", "RDEPEND", "PDEPEND", 'repository'] + aux_keys = ["DEPEND", "EAPI", "RDEPEND", "PDEPEND", 'repository'] aux_dict = dict(zip(aux_keys, mydb.aux_get(mycpv, aux_keys))) + eapi = aux_dict.pop('EAPI') repository = aux_dict.pop('repository') try: mycommands = myupdates[repository] @@ -272,6 +301,6 @@ def perform_global_updates(mycpv, mydb, myupdates): if not mycommands: return - updates = update_dbentries(mycommands, aux_dict) + updates = update_dbentries(mycommands, aux_dict, eapi=eapi) if updates: mydb.aux_update(mycpv, updates) diff --git a/pym/_emerge/MetadataRegen.py b/pym/_emerge/MetadataRegen.py index 07fea73c499..e82015fd12e 100644 --- a/pym/_emerge/MetadataRegen.py +++ b/pym/_emerge/MetadataRegen.py @@ -11,7 +11,7 @@ class MetadataRegen(PollScheduler): def __init__(self, portdb, cp_iter=None, consumer=None, max_jobs=None, max_load=None): - PollScheduler.__init__(self) + PollScheduler.__init__(self, main=True) self._portdb = portdb self._global_cleanse = False if cp_iter is None: @@ -78,12 +78,11 @@ class MetadataRegen(PollScheduler): cpv, ebuild_path, repo_path) if metadata is not None: if consumer is not None: - consumer(cpv, repo_path, metadata, ebuild_hash) + consumer(cpv, repo_path, metadata, ebuild_hash, True) continue yield EbuildMetadataPhase(cpv=cpv, ebuild_hash=ebuild_hash, - metadata_callback=portdb._metadata_callback, portdb=portdb, repo_path=repo_path, settings=portdb.doebuild_settings) @@ -177,7 +176,8 @@ class MetadataRegen(PollScheduler): self._consumer(metadata_process.cpv, metadata_process.repo_path, metadata_process.metadata, - metadata_process.ebuild_hash) + metadata_process.ebuild_hash, + metadata_process.eapi_supported) self._schedule() diff --git a/pym/_emerge/Package.py b/pym/_emerge/Package.py index e4d416c8c44..14d06944978 100644 --- a/pym/_emerge/Package.py +++ b/pym/_emerge/Package.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -8,10 +8,10 @@ from portage import _encodings, _unicode_decode, _unicode_encode from portage.cache.mappings import slot_dict_class from portage.const import EBUILD_PHASES from portage.dep import Atom, check_required_use, use_reduce, \ - paren_enclose, _slot_re, _slot_separator, _repo_separator -from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use + paren_enclose, _slot_separator, _repo_separator +from portage.versions import _pkg_str, _unknown_repo +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidDependString -from portage.repository.config import _gen_valid_repo from _emerge.Task import Task if sys.hexversion >= 0x3000000: @@ -25,9 +25,10 @@ class Package(Task): "installed", "metadata", "onlydeps", "operation", "root_config", "type_name", "category", "counter", "cp", "cpv_split", - "inherited", "invalid", "iuse", "masks", "mtime", - "pf", "pv_split", "root", "slot", "slot_atom", "visible",) + \ - ("_raw_metadata", "_use",) + "inherited", "iuse", "mtime", + "pf", "root", "slot", "slot_abi", "slot_atom", "version") + \ + ("_invalid", "_raw_metadata", "_masks", "_use", + "_validated_atoms", "_visible") metadata_keys = [ "BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "EAPI", @@ -38,7 +39,7 @@ class Package(Task): _dep_keys = ('DEPEND', 'PDEPEND', 'RDEPEND',) _use_conditional_misc_keys = ('LICENSE', 'PROPERTIES', 'RESTRICT') - UNKNOWN_REPO = "__unknown__" + UNKNOWN_REPO = _unknown_repo def __init__(self, **kwargs): Task.__init__(self, **kwargs) @@ -49,33 +50,30 @@ class Package(Task): self.metadata = _PackageMetadataWrapper(self, self._raw_metadata) if not self.built: self.metadata['CHOST'] = self.root_config.settings.get('CHOST', '') - self.cp = portage.cpv_getkey(self.cpv) - slot = self.slot - if _slot_re.match(slot) is None: + eapi_attrs = _get_eapi_attrs(self.metadata["EAPI"]) + self.cpv = _pkg_str(self.cpv, slot=self.metadata["SLOT"], + repo=self.metadata.get('repository', ''), + eapi=self.metadata["EAPI"]) + if hasattr(self.cpv, 'slot_invalid'): self._invalid_metadata('SLOT.invalid', - "SLOT: invalid value: '%s'" % slot) - # Avoid an InvalidAtom exception when creating slot_atom. - # This package instance will be masked due to empty SLOT. - slot = '0' + "SLOT: invalid value: '%s'" % self.metadata["SLOT"]) + self.cp = self.cpv.cp + self.slot = self.cpv.slot + self.slot_abi = self.cpv.slot_abi + # sync metadata with validated repo (may be UNKNOWN_REPO) + self.metadata['repository'] = self.cpv.repo if (self.iuse.enabled or self.iuse.disabled) and \ - not eapi_has_iuse_defaults(self.metadata["EAPI"]): + not eapi_attrs.iuse_defaults: if not self.installed: self._invalid_metadata('EAPI.incompatible', "IUSE contains defaults, but EAPI doesn't allow them") - self.slot_atom = portage.dep.Atom("%s%s%s" % (self.cp, _slot_separator, slot)) + self.slot_atom = Atom("%s%s%s" % (self.cp, _slot_separator, self.slot)) self.category, self.pf = portage.catsplit(self.cpv) - self.cpv_split = portage.catpkgsplit(self.cpv) - self.pv_split = self.cpv_split[1:] + self.cpv_split = self.cpv.cpv_split + self.version = self.cpv.version if self.inherited is None: self.inherited = frozenset() - repo = _gen_valid_repo(self.metadata.get('repository', '')) - if not repo: - repo = self.UNKNOWN_REPO - self.metadata['repository'] = repo - - self._validate_deps() - self.masks = self._masks() - self.visible = self._visible(self.masks) + if self.operation is None: if self.onlydeps or self.installed: self.operation = "nomerge" @@ -84,11 +82,44 @@ class Package(Task): self._hash_key = Package._gen_hash_key(cpv=self.cpv, installed=self.installed, onlydeps=self.onlydeps, - operation=self.operation, repo_name=repo, + operation=self.operation, repo_name=self.cpv.repo, root_config=self.root_config, type_name=self.type_name) self._hash_value = hash(self._hash_key) + # These are calculated on-demand, so that they are calculated + # after FakeVartree applies its metadata tweaks. + @property + def invalid(self): + if self._invalid is None: + self._validate_deps() + if self._invalid is None: + self._invalid = False + return self._invalid + + @property + def masks(self): + if self._masks is None: + self._masks = self._eval_masks() + return self._masks + + @property + def visible(self): + if self._visible is None: + self._visible = self._eval_visiblity(self.masks) + return self._visible + + @property + def validated_atoms(self): + """ + Returns *all* validated atoms from the deps, regardless + of USE conditionals, with USE conditionals inside + atoms left unevaluated. + """ + if self._validated_atoms is None: + self._validate_deps() + return self._validated_atoms + @classmethod def _gen_hash_key(cls, cpv=None, installed=None, onlydeps=None, operation=None, repo_name=None, root_config=None, @@ -142,16 +173,21 @@ class Package(Task): dep_eapi = None dep_valid_flag = None + validated_atoms = [] for k in self._dep_keys: v = self.metadata.get(k) if not v: continue try: - use_reduce(v, eapi=dep_eapi, matchall=True, - is_valid_flag=dep_valid_flag, token_class=Atom) + validated_atoms.extend(use_reduce(v, eapi=dep_eapi, + matchall=True, is_valid_flag=dep_valid_flag, + token_class=Atom, flat=True)) except InvalidDependString as e: self._metadata_exception(k, e) + self._validated_atoms = tuple(set(atom for atom in + validated_atoms if isinstance(atom, Atom))) + k = 'PROVIDE' v = self.metadata.get(k) if v: @@ -175,7 +211,7 @@ class Package(Task): k = 'REQUIRED_USE' v = self.metadata.get(k) if v: - if not eapi_has_required_use(eapi): + if not _get_eapi_attrs(eapi).required_use: self._invalid_metadata('EAPI.incompatible', "REQUIRED_USE set, but EAPI='%s' doesn't allow it" % eapi) else: @@ -205,11 +241,11 @@ class Package(Task): onlydeps=self.onlydeps, operation=self.operation, root_config=self.root_config, type_name=self.type_name) - def _masks(self): + def _eval_masks(self): masks = {} settings = self.root_config.settings - if self.invalid is not None: + if self.invalid is not False: masks['invalid'] = self.invalid if not settings._accept_chost(self.cpv, self.metadata): @@ -249,13 +285,13 @@ class Package(Task): pass if not masks: - masks = None + masks = False return masks - def _visible(self, masks): + def _eval_visiblity(self, masks): - if masks is not None: + if masks is not False: if 'EAPI.unsupported' in masks: return False @@ -338,12 +374,12 @@ class Package(Task): _unicode_decode("%s: %s in '%s'") % (k, e, path)) def _invalid_metadata(self, msg_type, msg): - if self.invalid is None: - self.invalid = {} - msgs = self.invalid.get(msg_type) + if self._invalid is None: + self._invalid = {} + msgs = self._invalid.get(msg_type) if msgs is None: msgs = [] - self.invalid[msg_type] = msgs + self._invalid[msg_type] = msgs msgs.append(msg) def __str__(self): @@ -529,28 +565,28 @@ class Package(Task): def __lt__(self, other): if other.cp != self.cp: return False - if portage.pkgcmp(self.pv_split, other.pv_split) < 0: + if portage.vercmp(self.version, other.version) < 0: return True return False def __le__(self, other): if other.cp != self.cp: return False - if portage.pkgcmp(self.pv_split, other.pv_split) <= 0: + if portage.vercmp(self.version, other.version) <= 0: return True return False def __gt__(self, other): if other.cp != self.cp: return False - if portage.pkgcmp(self.pv_split, other.pv_split) > 0: + if portage.vercmp(self.version, other.version) > 0: return True return False def __ge__(self, other): if other.cp != self.cp: return False - if portage.pkgcmp(self.pv_split, other.pv_split) >= 0: + if portage.vercmp(self.version, other.version) >= 0: return True return False @@ -568,7 +604,7 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): __slots__ = ("_pkg",) _wrapped_keys = frozenset( - ["COUNTER", "INHERITED", "IUSE", "SLOT", "USE", "_mtime_"]) + ["COUNTER", "INHERITED", "IUSE", "USE", "_mtime_"]) _use_conditional_keys = frozenset( ['LICENSE', 'PROPERTIES', 'PROVIDE', 'RESTRICT',]) @@ -641,9 +677,6 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): self._pkg.iuse = self._pkg._iuse( v.split(), self._pkg.root_config.settings._iuse_implicit_match) - def _set_slot(self, k, v): - self._pkg.slot = v - def _set_counter(self, k, v): if isinstance(v, basestring): try: diff --git a/pym/_emerge/PackageVirtualDbapi.py b/pym/_emerge/PackageVirtualDbapi.py index a692bb60241..0f7be44b1c6 100644 --- a/pym/_emerge/PackageVirtualDbapi.py +++ b/pym/_emerge/PackageVirtualDbapi.py @@ -1,8 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys from portage.dbapi import dbapi +from portage.dbapi.dep_expand import dep_expand class PackageVirtualDbapi(dbapi): """ @@ -76,20 +77,24 @@ class PackageVirtualDbapi(dbapi): self._match_cache = {} def match(self, origdep, use_cache=1): - result = self._match_cache.get(origdep) + atom = dep_expand(origdep, mydb=self, settings=self.settings) + cache_key = (atom, atom.unevaluated_atom) + result = self._match_cache.get(cache_key) if result is not None: return result[:] - result = dbapi.match(self, origdep, use_cache=use_cache) - self._match_cache[origdep] = result + result = list(self._iter_match(atom, self.cp_list(atom.cp))) + self._match_cache[cache_key] = result return result[:] def cpv_exists(self, cpv, myrepo=None): return cpv in self._cpv_map def cp_list(self, mycp, use_cache=1): - cachelist = self._match_cache.get(mycp) - # cp_list() doesn't expand old-style virtuals - if cachelist and cachelist[0].startswith(mycp): + # NOTE: Cache can be safely shared with the match cache, since the + # match cache uses the result from dep_expand for the cache_key. + cache_key = (mycp, mycp) + cachelist = self._match_cache.get(cache_key) + if cachelist is not None: return cachelist[:] cpv_list = self._cp_map.get(mycp) if cpv_list is None: @@ -97,8 +102,7 @@ class PackageVirtualDbapi(dbapi): else: cpv_list = [pkg.cpv for pkg in cpv_list] self._cpv_sort_ascending(cpv_list) - if not (not cpv_list and mycp.startswith("virtual/")): - self._match_cache[mycp] = cpv_list + self._match_cache[cache_key] = cpv_list return cpv_list[:] def cp_all(self): diff --git a/pym/_emerge/PollScheduler.py b/pym/_emerge/PollScheduler.py index 1c631c3f27b..5103e31d600 100644 --- a/pym/_emerge/PollScheduler.py +++ b/pym/_emerge/PollScheduler.py @@ -13,6 +13,7 @@ from portage import _encodings from portage import _unicode_encode from portage.util import writemsg_level from portage.util.SlotObject import SlotObject +from portage.util._eventloop.EventLoop import EventLoop from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.getloadavg import getloadavg @@ -26,7 +27,13 @@ class PollScheduler(object): "output", "register", "run", "source_remove", "timeout_add", "unregister") - def __init__(self): + def __init__(self, main=False): + """ + @param main: If True then use global_event_loop(), otherwise use + a local EventLoop instance (default is False, for safe use in + a non-main thread) + @type main: bool + """ self._terminated = threading.Event() self._terminated_tasks = False self._max_jobs = 1 @@ -34,7 +41,10 @@ class PollScheduler(object): self._jobs = 0 self._scheduling = False self._background = False - self._event_loop = global_event_loop() + if main: + self._event_loop = global_event_loop() + else: + self._event_loop = EventLoop(main=False) self.sched_iface = self._sched_iface_class( IO_ERR=self._event_loop.IO_ERR, IO_HUP=self._event_loop.IO_HUP, @@ -138,15 +148,22 @@ class PollScheduler(object): def _main_loop(self): term_check_id = self.sched_iface.idle_add(self._termination_check) try: - # Populate initial event sources. We only need to do - # this once here, since it can be called during the - # loop from within event handlers. + # Populate initial event sources. Unless we're scheduling + # based on load average, we only need to do this once + # here, since it can be called during the loop from within + # event handlers. self._schedule() + max_load = self._max_load # Loop while there are jobs to be scheduled. while self._keep_scheduling(): self.sched_iface.iteration() + if max_load is not None: + # We have to schedule periodically, in case the load + # average has changed since the last call. + self._schedule() + # Clean shutdown of previously scheduled jobs. In the # case of termination, this allows for basic cleanup # such as flushing of buffered output to logs. diff --git a/pym/_emerge/QueueScheduler.py b/pym/_emerge/QueueScheduler.py index 9d73b782631..206087c7af4 100644 --- a/pym/_emerge/QueueScheduler.py +++ b/pym/_emerge/QueueScheduler.py @@ -10,8 +10,8 @@ class QueueScheduler(PollScheduler): run() method returns when no tasks remain. """ - def __init__(self, max_jobs=None, max_load=None): - PollScheduler.__init__(self) + def __init__(self, main=True, max_jobs=None, max_load=None): + PollScheduler.__init__(self, main=main) if max_jobs is None: max_jobs = 1 diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index 8d6ab9fb8c5..0b72a4cfc17 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -135,8 +135,9 @@ class Scheduler(PollScheduler): portage.exception.PortageException.__init__(self, value) def __init__(self, settings, trees, mtimedb, myopts, - spinner, mergelist=None, favorites=None, graph_config=None): - PollScheduler.__init__(self) + spinner, mergelist=None, favorites=None, graph_config=None, + uninstall_only=False): + PollScheduler.__init__(self, main=True) if mergelist is not None: warnings.warn("The mergelist parameter of the " + \ @@ -151,6 +152,7 @@ class Scheduler(PollScheduler): self._spinner = spinner self._mtimedb = mtimedb self._favorites = favorites + self._uninstall_only = uninstall_only self._args_set = InternalPackageSet(favorites, allow_repo=True) self._build_opts = self._build_opts_class() @@ -326,10 +328,15 @@ class Scheduler(PollScheduler): self._set_graph_config(graph_config) self._blocker_db = {} dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_abi_deps = self.myopts.get( + "--ignore-built-slot-abi-deps", "n") == "y" for root in self.trees: + if self._uninstall_only: + continue if graph_config is None: fake_vartree = FakeVartree(self.trees[root]["root_config"], - pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps) + pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps, + ignore_built_slot_abi_deps=ignore_built_slot_abi_deps) fake_vartree.sync() else: fake_vartree = graph_config.trees[root]['vartree'] diff --git a/pym/_emerge/SetArg.py b/pym/_emerge/SetArg.py index 94cf0a6ceb4..5c829754760 100644 --- a/pym/_emerge/SetArg.py +++ b/pym/_emerge/SetArg.py @@ -1,9 +1,12 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.DependencyArg import DependencyArg from portage._sets import SETPREFIX class SetArg(DependencyArg): + + __slots__ = ('name', 'pset') + def __init__(self, pset=None, **kwargs): DependencyArg.__init__(self, **kwargs) self.pset = pset diff --git a/pym/_emerge/TaskScheduler.py b/pym/_emerge/TaskScheduler.py index 71ac80f1432..583bfe32394 100644 --- a/pym/_emerge/TaskScheduler.py +++ b/pym/_emerge/TaskScheduler.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.QueueScheduler import QueueScheduler @@ -11,9 +11,9 @@ class TaskScheduler(object): add tasks and call run(). The run() method returns when no tasks remain. """ - def __init__(self, max_jobs=None, max_load=None): + def __init__(self, main=True, max_jobs=None, max_load=None): self._queue = SequentialTaskQueue(max_jobs=max_jobs) - self._scheduler = QueueScheduler( + self._scheduler = QueueScheduler(main=main, max_jobs=max_jobs, max_load=max_load) self.sched_iface = self._scheduler.sched_iface self.run = self._scheduler.run diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 22c3e265f1a..9a023a84a11 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -28,15 +28,15 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage.localization import _ from portage import os from portage import shutil -from portage import _unicode_decode +from portage import eapi_is_supported, _unicode_decode from portage.cache.cache_errors import CacheError from portage.const import GLOBAL_CONFIG_PATH -from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_SET_CONFIG +from portage.const import _ENABLE_DYN_LINK_MAP from portage.dbapi.dep_expand import dep_expand from portage.dbapi._expand_new_virt import expand_new_virt -from portage.dep import Atom, extended_cp_match +from portage.dep import Atom from portage.eclass_cache import hashed_path -from portage.exception import InvalidAtom +from portage.exception import InvalidAtom, InvalidData from portage.output import blue, bold, colorize, create_color_func, darkgreen, \ red, yellow good = create_color_func("GOOD") @@ -76,6 +76,9 @@ from _emerge.userquery import userquery if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode def action_build(settings, trees, mtimedb, myopts, myaction, myfiles, spinner): @@ -172,6 +175,7 @@ def action_build(settings, trees, mtimedb, verbose = "--verbose" in myopts quiet = "--quiet" in myopts myparams = create_depgraph_params(myopts, myaction) + mergelist_shown = False if pretend or fetchonly: # make the mtimedb readonly @@ -319,6 +323,7 @@ def action_build(settings, trees, mtimedb, mydepgraph.altlist(reversed=tree), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval prompt="Would you like to resume merging these packages?" @@ -327,6 +332,7 @@ def action_build(settings, trees, mtimedb, mydepgraph.altlist(reversed=("--tree" in myopts)), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval mergecount=0 @@ -383,6 +389,7 @@ def action_build(settings, trees, mtimedb, mydepgraph.altlist(reversed=tree), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval else: @@ -390,6 +397,7 @@ def action_build(settings, trees, mtimedb, mydepgraph.altlist(reversed=("--tree" in myopts)), favorites=favorites) mydepgraph.display_problems() + mergelist_shown = True if retval != os.EX_OK: return retval if "--buildpkgonly" in myopts: @@ -420,6 +428,11 @@ def action_build(settings, trees, mtimedb, print("!!! Cannot merge requested packages. Merge deps and try again.\n") return 1 + if not mergelist_shown: + # If we haven't already shown the merge list above, at + # least show warnings about missed updates and such. + mydepgraph.display_problems() + if ("--resume" in myopts): favorites=mtimedb["resume"]["favorites"] @@ -1287,12 +1300,21 @@ def action_deselect(settings, trees, opts, atoms): break if discard_atoms: for atom in sorted(discard_atoms): + if pretend: - print(">>> Would remove %s from \"world\" favorites file..." % \ - colorize("INFORM", str(atom))) + action_desc = "Would remove" else: - print(">>> Removing %s from \"world\" favorites file..." % \ - colorize("INFORM", str(atom))) + action_desc = "Removing" + + if atom.startswith(SETPREFIX): + filename = "world_sets" + else: + filename = "world" + + writemsg_stdout( + ">>> %s %s from \"%s\" favorites file...\n" % + (action_desc, colorize("INFORM", _unicode(atom)), + filename), noiselevel=-1) if '--ask' in opts: prompt = "Would you like to remove these " + \ @@ -1456,11 +1478,11 @@ def action_info(settings, trees, myopts, myfiles): append("Repositories: %s" % \ " ".join(repo.name for repo in repos)) - if _ENABLE_SET_CONFIG: + installed_sets = sorted(s for s in + root_config.sets['selected'].getNonAtoms() if s.startswith(SETPREFIX)) + if installed_sets: sets_line = "Installed sets: " - sets_line += ", ".join(s for s in \ - sorted(root_config.sets['selected'].getNonAtoms()) \ - if s.startswith(SETPREFIX)) + sets_line += ", ".join(installed_sets) append(sets_line) if "--verbose" in myopts: @@ -1716,9 +1738,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None): if onProgress is not None: onProgress(maxval, curval) - from portage import eapi_is_supported, \ - _validate_cache_for_unsupported_eapis - # TODO: Display error messages, but do not interfere with the progress bar. # Here's how: # 1) erase the progress bar @@ -1755,11 +1774,9 @@ def action_metadata(settings, portdb, myopts, porttrees=None): eapi = src.get('EAPI') if not eapi: eapi = '0' - eapi = eapi.lstrip('-') eapi_supported = eapi_is_supported(eapi) if not eapi_supported: - if not _validate_cache_for_unsupported_eapis: - continue + continue dest = None try: @@ -1804,13 +1821,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None): # so there's no need to overwrite it. continue - if not eapi_supported: - src = { - 'EAPI' : '-' + eapi, - dest_chf_key : src[dest_chf_key], - '_eclasses_' : src['_eclasses_'], - } - try: tree_data.dest_db[cpv] = src except CacheError: @@ -2584,16 +2594,30 @@ def action_uninstall(settings, trees, ldpath_mtimes, level=logging.ERROR, noiselevel=-1) return 1 - for cp in vardb.cp_all(): - if extended_cp_match(ext_atom.cp, cp): - atom = cp + for cpv in vardb.cpv_all(): + if portage.match_from_list(ext_atom, [cpv]): + require_metadata = False + atom = portage.cpv_getkey(cpv) + if ext_atom.operator == '=*': + atom = "=" + atom + "-" + \ + portage.versions.cpv_getversion(cpv) if ext_atom.slot: atom += ":" + ext_atom.slot + require_metadata = True if ext_atom.repo: atom += "::" + ext_atom.repo + require_metadata = True - if vardb.match(atom): - valid_atoms.append(Atom(atom, allow_repo=True)) + atom = Atom(atom, allow_repo=True) + if require_metadata: + try: + cpv = vardb._pkg_str(cpv, ext_atom.repo) + except (KeyError, InvalidData): + continue + if not portage.match_from_list(atom, [cpv]): + continue + + valid_atoms.append(atom) else: msg = [] @@ -2657,7 +2681,7 @@ def action_uninstall(settings, trees, ldpath_mtimes, # redirection of ebuild phase output to logs as required for # options such as --quiet. sched = Scheduler(settings, trees, None, opts, - spinner) + spinner, uninstall_only=True) sched._background = sched._background_mode() sched._status_display.quiet = True @@ -2809,6 +2833,7 @@ def relative_profile_path(portdir, abs_profile): def getportageversion(portdir, _unused, profile, chost, vardb): profilever = None + repositories = vardb.settings.repositories if profile: profilever = relative_profile_path(portdir, profile) if profilever is None: @@ -2819,6 +2844,20 @@ def getportageversion(portdir, _unused, profile, chost, vardb): os.path.join(profile, parent)) if profilever is not None: break + colon = parent.find(":") + if colon != -1: + p_repo_name = parent[:colon] + try: + p_repo_loc = \ + repositories.get_location_for_name(p_repo_name) + except KeyError: + pass + else: + profilever = relative_profile_path(p_repo_loc, + os.path.join(p_repo_loc, 'profiles', + parent[colon+1:])) + if profilever is not None: + break except portage.exception.PortageException: pass diff --git a/pym/_emerge/create_depgraph_params.py b/pym/_emerge/create_depgraph_params.py index 8f15c681312..2838e93c3d7 100644 --- a/pym/_emerge/create_depgraph_params.py +++ b/pym/_emerge/create_depgraph_params.py @@ -15,12 +15,22 @@ def create_depgraph_params(myopts, myaction): # complete: completely account for all known dependencies # remove: build graph for use in removing packages # rebuilt_binaries: replace installed packages with rebuilt binaries + # rebuild_if_new_slot_abi: rebuild or reinstall packages when + # SLOT/ABI := operator dependencies can be satisfied by a newer + # SLOT/ABI, so that older packages slots will become eligible for + # removal by the --depclean action as soon as possible + # ignore_built_slot_abi_deps: ignore the SLOT/ABI := operator parts + # of dependencies that have been recorded when packages where built myparams = {"recurse" : True} bdeps = myopts.get("--with-bdeps") if bdeps is not None: myparams["bdeps"] = bdeps + ignore_built_slot_abi_deps = myopts.get("--ignore-built-slot-abi-deps") + if ignore_built_slot_abi_deps is not None: + myparams["ignore_built_slot_abi_deps"] = ignore_built_slot_abi_deps + dynamic_deps = myopts.get("--dynamic-deps") if dynamic_deps is not None: myparams["dynamic_deps"] = dynamic_deps @@ -31,6 +41,10 @@ def create_depgraph_params(myopts, myaction): myparams["selective"] = True return myparams + rebuild_if_new_slot_abi = myopts.get('--rebuild-if-new-slot-abi') + if rebuild_if_new_slot_abi is not None: + myparams['rebuild_if_new_slot_abi'] = rebuild_if_new_slot_abi + if "--update" in myopts or \ "--newuse" in myopts or \ "--reinstall" in myopts or \ @@ -42,6 +56,11 @@ def create_depgraph_params(myopts, myaction): if deep is not None and deep != 0: myparams["deep"] = deep + complete_if_new_use = \ + myopts.get("--complete-graph-if-new-use") + if complete_if_new_use is not None: + myparams["complete_if_new_use"] = complete_if_new_use + complete_if_new_ver = \ myopts.get("--complete-graph-if-new-ver") if complete_if_new_ver is not None: diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index c01dc72dae0..0f3bc9389c9 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -18,10 +18,14 @@ from portage import os, OrderedDict from portage import _unicode_decode, _unicode_encode, _encodings from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH from portage.dbapi import dbapi +from portage.dbapi.dep_expand import dep_expand from portage.dep import Atom, best_match_to_list, extract_affecting_use, \ - check_required_use, human_readable_required_use, _repo_separator + check_required_use, human_readable_required_use, match_from_list, \ + _repo_separator +from portage.dep._slot_abi import ignore_built_slot_abi_deps from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use -from portage.exception import InvalidAtom, InvalidDependString, PortageException +from portage.exception import (InvalidAtom, InvalidDependString, + PackageNotFound, PortageException) from portage.output import colorize, create_color_func, \ darkgreen, green bad = create_color_func("BAD") @@ -71,6 +75,9 @@ from _emerge.resolver.output import Display if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class _scheduler_graph_config(object): def __init__(self, trees, pkg_cache, graph, mergelist): @@ -83,9 +90,9 @@ def _wildcard_set(atoms): pkgs = InternalPackageSet(allow_wildcard=True) for x in atoms: try: - x = Atom(x, allow_wildcard=True) + x = Atom(x, allow_wildcard=True, allow_repo=False) except portage.exception.InvalidAtom: - x = Atom("*/" + x, allow_wildcard=True) + x = Atom("*/" + x, allow_wildcard=True, allow_repo=False) pkgs.add(x) return pkgs @@ -108,6 +115,8 @@ class _frozen_depgraph_config(object): self._pkg_cache = {} self._highest_license_masked = {} dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" + ignore_built_slot_abi_deps = myopts.get( + "--ignore-built-slot-abi-deps", "n") == "y" for myroot in trees: self.trees[myroot] = {} # Create a RootConfig instance that references @@ -122,7 +131,8 @@ class _frozen_depgraph_config(object): FakeVartree(trees[myroot]["root_config"], pkg_cache=self._pkg_cache, pkg_root_config=self.roots[myroot], - dynamic_deps=dynamic_deps) + dynamic_deps=dynamic_deps, + ignore_built_slot_abi_deps=ignore_built_slot_abi_deps) self.pkgsettings[myroot] = portage.config( clone=self.trees[myroot]["vartree"].settings) @@ -364,12 +374,15 @@ class _dynamic_depgraph_config(object): # This use used to check if we have accounted for blockers # relevant to a package. self._traversed_pkg_deps = set() - self._slot_collision_info = {} + # This should be ordered such that the backtracker will + # attempt to solve conflicts which occurred earlier first, + # since an earlier conflict can be the cause of a conflict + # which occurs later. + self._slot_collision_info = OrderedDict() # Slot collision nodes are not allowed to block other packages since # blocker validation is only able to account for one package per slot. self._slot_collision_nodes = set() self._parent_atoms = {} - self._slot_conflict_parent_atoms = set() self._slot_conflict_handler = None self._circular_dependency_handler = None self._serialized_tasks_cache = None @@ -399,6 +412,7 @@ class _dynamic_depgraph_config(object): self._needed_license_changes = backtrack_parameters.needed_license_changes self._needed_use_config_changes = backtrack_parameters.needed_use_config_changes self._runtime_pkg_mask = backtrack_parameters.runtime_pkg_mask + self._slot_abi_replace_installed = backtrack_parameters.slot_abi_replace_installed self._need_restart = False # For conditions that always require user intervention, such as # unsatisfied REQUIRED_USE (currently has no autounmask support). @@ -408,6 +422,8 @@ class _dynamic_depgraph_config(object): self._autounmask = depgraph._frozen_config.myopts.get('--autounmask') != 'n' self._success_without_autounmask = False self._traverse_ignored_deps = False + self._complete_mode = False + self._slot_abi_deps = {} for myroot in depgraph._frozen_config.trees: self.sets[myroot] = _depgraph_sets() @@ -519,6 +535,10 @@ class depgraph(object): preload_installed_pkgs = \ "--nodeps" not in self._frozen_config.myopts + if self._frozen_config.myopts.get("--root-deps") is not None and \ + myroot != self._frozen_config.target_root: + continue + fake_vartree = self._frozen_config.trees[myroot]["vartree"] if not fake_vartree.dbapi: # This needs to be called for the first depgraph, but not for @@ -609,7 +629,7 @@ class depgraph(object): for line in msg: if line: line = colorize("INFORM", line) - writemsg_stdout(line + "\n", noiselevel=-1) + writemsg(line + "\n", noiselevel=-1) def _show_missed_update(self): @@ -795,37 +815,400 @@ class depgraph(object): def _process_slot_conflicts(self): """ + If there are any slot conflicts and backtracking is enabled, + _complete_graph should complete the graph before this method + is called, so that all relevant reverse dependencies are + available for use in backtracking decisions. + """ + for (slot_atom, root), slot_nodes in \ + self._dynamic_config._slot_collision_info.items(): + self._process_slot_conflict(root, slot_atom, slot_nodes) + + def _process_slot_conflict(self, root, slot_atom, slot_nodes): + """ Process slot conflict data to identify specific atoms which lead to conflict. These atoms only match a subset of the packages that have been pulled into a given slot. """ - for (slot_atom, root), slot_nodes \ - in self._dynamic_config._slot_collision_info.items(): - all_parent_atoms = set() - for pkg in slot_nodes: - parent_atoms = self._dynamic_config._parent_atoms.get(pkg) - if not parent_atoms: + debug = "--debug" in self._frozen_config.myopts + + slot_parent_atoms = set() + for pkg in slot_nodes: + parent_atoms = self._dynamic_config._parent_atoms.get(pkg) + if not parent_atoms: + continue + slot_parent_atoms.update(parent_atoms) + + conflict_pkgs = [] + conflict_atoms = {} + for pkg in slot_nodes: + + if self._dynamic_config._allow_backtracking and \ + pkg in self._dynamic_config._runtime_pkg_mask: + if debug: + writemsg_level( + "!!! backtracking loop detected: %s %s\n" % \ + (pkg, + self._dynamic_config._runtime_pkg_mask[pkg]), + level=logging.DEBUG, noiselevel=-1) + + parent_atoms = self._dynamic_config._parent_atoms.get(pkg) + if parent_atoms is None: + parent_atoms = set() + self._dynamic_config._parent_atoms[pkg] = parent_atoms + + all_match = True + for parent_atom in slot_parent_atoms: + if parent_atom in parent_atoms: continue - all_parent_atoms.update(parent_atoms) + # Use package set for matching since it will match via + # PROVIDE when necessary, while match_from_list does not. + parent, atom = parent_atom + atom_set = InternalPackageSet( + initial_atoms=(atom,), allow_repo=True) + if atom_set.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + parent_atoms.add(parent_atom) + else: + all_match = False + conflict_atoms.setdefault(parent_atom, set()).add(pkg) - for pkg in slot_nodes: - parent_atoms = self._dynamic_config._parent_atoms.get(pkg) - if parent_atoms is None: - parent_atoms = set() - self._dynamic_config._parent_atoms[pkg] = parent_atoms - for parent_atom in all_parent_atoms: - if parent_atom in parent_atoms: + if not all_match: + conflict_pkgs.append(pkg) + + if conflict_pkgs and \ + self._dynamic_config._allow_backtracking and \ + not self._accept_blocker_conflicts(): + remaining = [] + for pkg in conflict_pkgs: + if self._slot_conflict_backtrack_abi(pkg, + slot_nodes, conflict_atoms): + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + config.setdefault("slot_conflict_abi", set()).add(pkg) + else: + remaining.append(pkg) + if remaining: + self._slot_confict_backtrack(root, slot_atom, + slot_parent_atoms, remaining) + + def _slot_confict_backtrack(self, root, slot_atom, + all_parents, conflict_pkgs): + + debug = "--debug" in self._frozen_config.myopts + existing_node = self._dynamic_config._slot_pkg_map[root][slot_atom] + backtrack_data = [] + # The ordering of backtrack_data can make + # a difference here, because both mask actions may lead + # to valid, but different, solutions and the one with + # 'existing_node' masked is usually the better one. Because + # of that, we choose an order such that + # the backtracker will first explore the choice with + # existing_node masked. The backtracker reverses the + # order, so the order it uses is the reverse of the + # order shown here. See bug #339606. + if existing_node in conflict_pkgs and \ + existing_node is not conflict_pkgs[-1]: + conflict_pkgs.remove(existing_node) + conflict_pkgs.append(existing_node) + for to_be_masked in conflict_pkgs: + # For missed update messages, find out which + # atoms matched to_be_selected that did not + # match to_be_masked. + parent_atoms = \ + self._dynamic_config._parent_atoms.get(to_be_masked, set()) + conflict_atoms = set(parent_atom for parent_atom in all_parents \ + if parent_atom not in parent_atoms) + backtrack_data.append((to_be_masked, conflict_atoms)) + + if len(backtrack_data) > 1: + # NOTE: Generally, we prefer to mask the higher + # version since this solves common cases in which a + # lower version is needed so that all dependencies + # will be satisfied (bug #337178). However, if + # existing_node happens to be installed then we + # mask that since this is a common case that is + # triggered when --update is not enabled. + if existing_node.installed: + pass + elif any(pkg > existing_node for pkg in conflict_pkgs): + backtrack_data.reverse() + + to_be_masked = backtrack_data[-1][0] + + self._dynamic_config._backtrack_infos.setdefault( + "slot conflict", []).append(backtrack_data) + self._dynamic_config._need_restart = True + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to slot conflict:") + msg.append(" first package: %s" % existing_node) + msg.append(" package to mask: %s" % to_be_masked) + msg.append(" slot: %s" % slot_atom) + msg.append(" parents: %s" % ", ".join( \ + "(%s, '%s')" % (ppkg, atom) for ppkg, atom in all_parents)) + msg.append("") + writemsg_level("".join("%s\n" % l for l in msg), + noiselevel=-1, level=logging.DEBUG) + + def _slot_conflict_backtrack_abi(self, pkg, slot_nodes, conflict_atoms): + """ + If one or more conflict atoms have a SLOT/ABI dep that can be resolved + by rebuilding the parent package, then schedule the rebuild via + backtracking, and return True. Otherwise, return False. + """ + + found_update = False + for parent_atom, conflict_pkgs in conflict_atoms.items(): + parent, atom = parent_atom + if atom.slot_abi_op != "=" or not parent.built: + continue + + if pkg not in conflict_pkgs: + continue + + for other_pkg in slot_nodes: + if other_pkg in conflict_pkgs: + continue + + dep = Dependency(atom=atom, child=other_pkg, + parent=parent, root=pkg.root) + + if self._slot_abi_update_probe(dep): + self._slot_abi_update_backtrack(dep) + found_update = True + + return found_update + + def _slot_abi_update_backtrack(self, dep, new_child_slot=None): + if new_child_slot is None: + child = dep.child + else: + child = new_child_slot + if "--debug" in self._frozen_config.myopts: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to missed slot abi update:") + msg.append(" child package: %s" % child) + if new_child_slot is not None: + msg.append(" new child slot package: %s" % new_child_slot) + msg.append(" parent package: %s" % dep.parent) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + backtrack_infos = self._dynamic_config._backtrack_infos + config = backtrack_infos.setdefault("config", {}) + + # mask unwanted binary packages if necessary + abi_masks = {} + if new_child_slot is None: + if not child.installed: + abi_masks.setdefault(child, {})["slot_abi_mask_built"] = None + if not dep.parent.installed: + abi_masks.setdefault(dep.parent, {})["slot_abi_mask_built"] = None + if abi_masks: + config.setdefault("slot_abi_mask_built", {}).update(abi_masks) + + # trigger replacement of installed packages if necessary + abi_reinstalls = set() + if dep.parent.installed: + abi_reinstalls.add((dep.parent.root, dep.parent.slot_atom)) + if new_child_slot is None and child.installed: + abi_reinstalls.add((child.root, child.slot_atom)) + if abi_reinstalls: + config.setdefault("slot_abi_replace_installed", + set()).update(abi_reinstalls) + + self._dynamic_config._need_restart = True + + def _slot_abi_update_probe(self, dep, new_child_slot=False): + """ + SLOT/ABI := operators tend to prevent updates from getting pulled in, + since installed packages pull in packages with the SLOT/ABI that they + were built against. Detect this case so that we can schedule rebuilds + and reinstalls when appropriate. + NOTE: This function only searches for updates that involve upgrades + to higher versions, since the logic required to detect when a + downgrade would be desirable is not implemented. + """ + + if dep.child.installed and \ + self._frozen_config.excluded_pkgs.findAtomForPackage(dep.child, + modified_use=self._pkg_use_enabled(dep.child)): + return None + + if dep.parent.installed and \ + self._frozen_config.excluded_pkgs.findAtomForPackage(dep.parent, + modified_use=self._pkg_use_enabled(dep.parent)): + return None + + debug = "--debug" in self._frozen_config.myopts + want_downgrade = None + + for replacement_parent in self._iter_similar_available(dep.parent, + dep.parent.slot_atom): + + for atom in replacement_parent.validated_atoms: + if not atom.slot_abi_op == "=" or \ + atom.blocker or \ + atom.cp != dep.atom.cp: + continue + + # Discard USE deps, we're only searching for an approximate + # pattern, and dealing with USE states is too complex for + # this purpose. + atom = atom.without_use + + if replacement_parent.built and \ + portage.dep._match_slot(atom, dep.child): + # Our selected replacement_parent appears to be built + # for the existing child selection. So, discard this + # parent and search for another. + break + + for pkg in self._iter_similar_available( + dep.child, atom): + if pkg.slot == dep.child.slot and \ + pkg.slot_abi == dep.child.slot_abi: + # If SLOT/ABI is identical, then there's + # no point in updating. continue - # Use package set for matching since it will match via - # PROVIDE when necessary, while match_from_list does not. - parent, atom = parent_atom - atom_set = InternalPackageSet( - initial_atoms=(atom,), allow_repo=True) - if atom_set.findAtomForPackage(pkg, modified_use=self._pkg_use_enabled(pkg)): - parent_atoms.add(parent_atom) + if new_child_slot: + if pkg.slot == dep.child.slot: + continue + if pkg < dep.child: + # the new slot only matters if the + # package version is higher + continue else: - self._dynamic_config._slot_conflict_parent_atoms.add(parent_atom) + if pkg.slot != dep.child.slot: + continue + if pkg < dep.child: + if want_downgrade is None: + want_downgrade = self._downgrade_probe(dep.child) + # be careful not to trigger a rebuild when + # the only version available with a + # different slot_abi is an older version + if not want_downgrade: + continue + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_abi_update_probe:") + msg.append(" existing child package: %s" % dep.child) + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" new child package: %s" % pkg) + msg.append(" new parent package: %s" % replacement_parent) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return pkg + + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("slot_abi_update_probe:") + msg.append(" existing child package: %s" % dep.child) + msg.append(" existing parent package: %s" % dep.parent) + msg.append(" new child package: %s" % None) + msg.append(" new parent package: %s" % None) + msg.append("") + writemsg_level("\n".join(msg), + noiselevel=-1, level=logging.DEBUG) + + return None + + def _downgrade_probe(self, pkg): + """ + Detect cases where a downgrade of the given package is considered + desirable due to the current version being masked or unavailable. + """ + available_pkg = None + for available_pkg in self._iter_similar_available(pkg, + pkg.slot_atom): + if available_pkg >= pkg: + # There's an available package of the same or higher + # version, so downgrade seems undesirable. + return False + + return available_pkg is not None + + def _iter_similar_available(self, graph_pkg, atom): + """ + Given a package that's in the graph, do a rough check to + see if a similar package is available to install. The given + graph_pkg itself may be yielded only if it's not installed. + """ + + usepkgonly = "--usepkgonly" in self._frozen_config.myopts + useoldpkg_atoms = self._frozen_config.useoldpkg_atoms + use_ebuild_visibility = self._frozen_config.myopts.get( + '--use-ebuild-visibility', 'n') != 'n' + + for pkg in self._iter_match_pkgs_any( + graph_pkg.root_config, atom): + if pkg.cp != graph_pkg.cp: + # discard old-style virtual match + continue + if pkg.installed: + continue + if pkg in self._dynamic_config._runtime_pkg_mask: + continue + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + continue + if not self._pkg_visibility_check(pkg): + continue + if pkg.built: + if self._equiv_binary_installed(pkg): + continue + if not (not use_ebuild_visibility and + (usepkgonly or useoldpkg_atoms.findAtomForPackage( + pkg, modified_use=self._pkg_use_enabled(pkg)))) and \ + not self._equiv_ebuild_visible(pkg): + continue + yield pkg + + def _slot_abi_trigger_reinstalls(self): + """ + Search for packages with slot-abi deps on older slots, and schedule + rebuilds if they can link to a newer slot that's in the graph. + """ + + rebuild_if_new_slot_abi = self._dynamic_config.myparams.get( + "rebuild_if_new_slot_abi", "y") == "y" + + for slot_key, slot_info in self._dynamic_config._slot_abi_deps.items(): + + for dep in slot_info: + if not (dep.child.built and dep.parent and + isinstance(dep.parent, Package) and dep.parent.built): + continue + + # Check for slot update first, since we don't want to + # trigger reinstall of the child package when a newer + # slot will be used instead. + if rebuild_if_new_slot_abi: + new_child = self._slot_abi_update_probe(dep, + new_child_slot=True) + if new_child: + self._slot_abi_update_backtrack(dep, + new_child_slot=new_child) + break + + if dep.want_update: + if self._slot_abi_update_probe(dep): + self._slot_abi_update_backtrack(dep) + break def _reinstall_for_flags(self, pkg, forced_flags, orig_use, orig_iuse, cur_use, cur_iuse): @@ -1113,7 +1496,8 @@ class depgraph(object): if atom is None: atom = Atom("=" + pkg.cpv) self._dynamic_config._unsatisfied_deps_for_display.append( - ((pkg.root, atom), {"myparent":dep.parent})) + ((pkg.root, atom), + {"myparent" : dep.parent, "show_req_use" : pkg})) self._dynamic_config._skip_restart = True return 0 @@ -1144,121 +1528,7 @@ class depgraph(object): (dep.parent, dep.atom)) return 1 else: - # A slot conflict has occurred. - # The existing node should not already be in - # runtime_pkg_mask, since that would trigger an - # infinite backtracking loop. - if self._dynamic_config._allow_backtracking and \ - existing_node in \ - self._dynamic_config._runtime_pkg_mask: - if "--debug" in self._frozen_config.myopts: - writemsg( - "!!! backtracking loop detected: %s %s\n" % \ - (existing_node, - self._dynamic_config._runtime_pkg_mask[ - existing_node]), noiselevel=-1) - elif self._dynamic_config._allow_backtracking and \ - not self._accept_blocker_conflicts() and \ - not self.need_restart(): - - self._add_slot_conflict(pkg) - if dep.atom is not None and dep.parent is not None: - self._add_parent_atom(pkg, (dep.parent, dep.atom)) - - if arg_atoms: - for parent_atom in arg_atoms: - parent, atom = parent_atom - self._add_parent_atom(pkg, parent_atom) - self._process_slot_conflicts() - - backtrack_data = [] - fallback_data = [] - all_parents = set() - # The ordering of backtrack_data can make - # a difference here, because both mask actions may lead - # to valid, but different, solutions and the one with - # 'existing_node' masked is usually the better one. Because - # of that, we choose an order such that - # the backtracker will first explore the choice with - # existing_node masked. The backtracker reverses the - # order, so the order it uses is the reverse of the - # order shown here. See bug #339606. - for to_be_selected, to_be_masked in (existing_node, pkg), (pkg, existing_node): - # For missed update messages, find out which - # atoms matched to_be_selected that did not - # match to_be_masked. - parent_atoms = \ - self._dynamic_config._parent_atoms.get(to_be_selected, set()) - if parent_atoms: - conflict_atoms = self._dynamic_config._slot_conflict_parent_atoms.intersection(parent_atoms) - if conflict_atoms: - parent_atoms = conflict_atoms - - all_parents.update(parent_atoms) - - all_match = True - for parent, atom in parent_atoms: - i = InternalPackageSet(initial_atoms=(atom,), - allow_repo=True) - if not i.findAtomForPackage(to_be_masked): - all_match = False - break - - fallback_data.append((to_be_masked, parent_atoms)) - - if all_match: - # 'to_be_masked' does not violate any parent atom, which means - # there is no point in masking it. - pass - else: - backtrack_data.append((to_be_masked, parent_atoms)) - - if not backtrack_data: - # This shouldn't happen, but fall back to the old - # behavior if this gets triggered somehow. - backtrack_data = fallback_data - - if len(backtrack_data) > 1: - # NOTE: Generally, we prefer to mask the higher - # version since this solves common cases in which a - # lower version is needed so that all dependencies - # will be satisfied (bug #337178). However, if - # existing_node happens to be installed then we - # mask that since this is a common case that is - # triggered when --update is not enabled. - if existing_node.installed: - pass - elif pkg > existing_node: - backtrack_data.reverse() - - to_be_masked = backtrack_data[-1][0] - - self._dynamic_config._backtrack_infos["slot conflict"] = backtrack_data - self._dynamic_config._need_restart = True - if "--debug" in self._frozen_config.myopts: - msg = [] - msg.append("") - msg.append("") - msg.append("backtracking due to slot conflict:") - if backtrack_data is fallback_data: - msg.append("!!! backtrack_data fallback") - msg.append(" first package: %s" % existing_node) - msg.append(" second package: %s" % pkg) - msg.append(" package to mask: %s" % to_be_masked) - msg.append(" slot: %s" % pkg.slot_atom) - msg.append(" parents: %s" % ", ".join( \ - "(%s, '%s')" % (ppkg, atom) for ppkg, atom in all_parents)) - msg.append("") - writemsg_level("".join("%s\n" % l for l in msg), - noiselevel=-1, level=logging.DEBUG) - return 0 - - # A slot collision has occurred. Sometimes this coincides - # with unresolvable blockers, so the slot collision will be - # shown later if there are no unresolvable blockers. self._add_slot_conflict(pkg) - slot_collision = True - if debug: writemsg_level( "%s%s %s\n" % ("Slot Conflict:".ljust(15), @@ -1267,6 +1537,8 @@ class depgraph(object): modified_use=self._pkg_use_enabled(existing_node))), level=logging.DEBUG, noiselevel=-1) + slot_collision = True + if slot_collision: # Now add this node to the graph so that self.display() # can show use flags and --tree portage.output. This node is @@ -1323,10 +1595,27 @@ class depgraph(object): # Installing package A, we need to make sure package A's deps are met. # emerge --deep <pkgspec>; we need to recursively check dependencies of pkgspec # If we are in --nodeps (no recursion) mode, we obviously only check 1 level of dependencies. - if arg_atoms: - depth = 0 + if arg_atoms and depth > 0: + for parent, atom in arg_atoms: + if parent.reset_depth: + depth = 0 + break + + if previously_added and pkg.depth is not None: + depth = min(pkg.depth, depth) pkg.depth = depth deep = self._dynamic_config.myparams.get("deep", 0) + update = "--update" in self._frozen_config.myopts + + dep.want_update = (not self._dynamic_config._complete_mode and + (arg_atoms or update) and + not (deep is not True and depth > deep)) + + dep.child = pkg + if (not pkg.onlydeps and pkg.built and + dep.atom and dep.atom.slot_abi_built): + self._add_slot_abi_dep(dep) + recurse = deep is True or depth + 1 <= deep dep_stack = self._dynamic_config._dep_stack if "recurse" not in self._dynamic_config.myparams: @@ -1358,6 +1647,14 @@ class depgraph(object): self._dynamic_config._parent_atoms[pkg] = parent_atoms parent_atoms.add(parent_atom) + def _add_slot_abi_dep(self, dep): + slot_key = (dep.root, dep.child.slot_atom) + slot_info = self._dynamic_config._slot_abi_deps.get(slot_key) + if slot_info is None: + slot_info = [] + self._dynamic_config._slot_abi_deps[slot_key] = slot_info + slot_info.append(dep) + def _add_slot_conflict(self, pkg): self._dynamic_config._slot_collision_nodes.add(pkg) slot_key = (pkg.slot_atom, pkg.root) @@ -1449,7 +1746,10 @@ class depgraph(object): try: dep_string = portage.dep.use_reduce(dep_string, - uselist=self._pkg_use_enabled(pkg), is_valid_flag=pkg.iuse.is_valid_flag) + uselist=self._pkg_use_enabled(pkg), + is_valid_flag=pkg.iuse.is_valid_flag, + opconvert=True, token_class=Atom, + eapi=pkg.metadata['EAPI']) except portage.exception.InvalidDependString as e: if not pkg.installed: # should have been masked before it was selected @@ -1461,7 +1761,9 @@ class depgraph(object): # practical to ignore this issue for installed packages. try: dep_string = portage.dep.use_reduce(dep_string, - uselist=self._pkg_use_enabled(pkg)) + uselist=self._pkg_use_enabled(pkg), + opconvert=True, token_class=Atom, + eapi=pkg.metadata['EAPI']) except portage.exception.InvalidDependString as e: self._dynamic_config._masked_installed.add(pkg) del e @@ -1482,9 +1784,6 @@ class depgraph(object): if not dep_string: continue - dep_string = portage.dep.paren_enclose(dep_string, - unevaluated_atom=True) - if not self._add_pkg_dep_string( pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): @@ -1518,7 +1817,9 @@ class depgraph(object): if debug: writemsg_level("\nParent: %s\n" % (pkg,), noiselevel=-1, level=logging.DEBUG) - writemsg_level("Depstring: %s\n" % (dep_string,), + dep_repr = portage.dep.paren_enclose(dep_string, + unevaluated_atom=True, opconvert=True) + writemsg_level("Depstring: %s\n" % (dep_repr,), noiselevel=-1, level=logging.DEBUG) writemsg_level("Priority: %s\n" % (dep_priority,), noiselevel=-1, level=logging.DEBUG) @@ -1596,16 +1897,11 @@ class depgraph(object): self._dynamic_config._slot_pkg_map[dep.child.root].get( dep.child.slot_atom) is None: myarg = None - if dep.root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(dep.child)) - except StopIteration: - pass - except InvalidDependString: - if not dep.child.installed: - # This shouldn't happen since the package - # should have been masked. - raise + try: + myarg = next(self._iter_atoms_for_pkg(dep.child), None) + except InvalidDependString: + if not dep.child.installed: + raise if myarg is None: # Existing child selection may not be valid unless @@ -1711,14 +2007,11 @@ class depgraph(object): self._dynamic_config._slot_pkg_map[dep.child.root].get( dep.child.slot_atom) is None: myarg = None - if dep.root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(dep.child)) - except StopIteration: - pass - except InvalidDependString: - if not dep.child.installed: - raise + try: + myarg = next(self._iter_atoms_for_pkg(dep.child), None) + except InvalidDependString: + if not dep.child.installed: + raise if myarg is None: ignored = True @@ -1812,9 +2105,14 @@ class depgraph(object): # Yield ~, =*, < and <= atoms first, since those are more likely to # cause slot conflicts, and we want those atoms to be displayed # in the resulting slot conflict message (see bug #291142). + # Give similar treatment to SLOT/ABI atoms. conflict_atoms = [] normal_atoms = [] + abi_atoms = [] for atom in cp_atoms: + if atom.slot_abi_built: + abi_atoms.append(atom) + continue conflict = False for child_pkg in atom_pkg_graph.child_nodes(atom): existing_node, matches = \ @@ -1827,7 +2125,7 @@ class depgraph(object): else: normal_atoms.append(atom) - for atom in chain(conflict_atoms, normal_atoms): + for atom in chain(abi_atoms, conflict_atoms, normal_atoms): child_pkgs = atom_pkg_graph.child_nodes(atom) # if more than one child, yield highest version if len(child_pkgs) > 1: @@ -1840,34 +2138,22 @@ class depgraph(object): Yields non-disjunctive deps. Raises InvalidDependString when necessary. """ - i = 0 - while i < len(dep_struct): - x = dep_struct[i] + for x in dep_struct: if isinstance(x, list): - for y in self._queue_disjunctive_deps( - pkg, dep_root, dep_priority, x): - yield y - elif x == "||": - self._queue_disjunction(pkg, dep_root, dep_priority, - [ x, dep_struct[ i + 1 ] ] ) - i += 1 + if x and x[0] == "||": + self._queue_disjunction(pkg, dep_root, dep_priority, [x]) + else: + for y in self._queue_disjunctive_deps( + pkg, dep_root, dep_priority, x): + yield y else: - try: - x = portage.dep.Atom(x, eapi=pkg.metadata["EAPI"]) - except portage.exception.InvalidAtom: - if not pkg.installed: - raise portage.exception.InvalidDependString( - "invalid atom: '%s'" % x) + # Note: Eventually this will check for PROPERTIES=virtual + # or whatever other metadata gets implemented for this + # purpose. + if x.cp.startswith('virtual/'): + self._queue_disjunction(pkg, dep_root, dep_priority, [x]) else: - # Note: Eventually this will check for PROPERTIES=virtual - # or whatever other metadata gets implemented for this - # purpose. - if x.cp.startswith('virtual/'): - self._queue_disjunction( pkg, dep_root, - dep_priority, [ str(x) ] ) - else: - yield str(x) - i += 1 + yield x def _queue_disjunction(self, pkg, dep_root, dep_priority, dep_struct): self._dynamic_config._dep_disjunctive_stack.append( @@ -1880,10 +2166,8 @@ class depgraph(object): """ pkg, dep_root, dep_priority, dep_struct = \ self._dynamic_config._dep_disjunctive_stack.pop() - dep_string = portage.dep.paren_enclose(dep_struct, - unevaluated_atom=True) if not self._add_pkg_dep_string( - pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): + pkg, dep_root, dep_priority, dep_struct, allow_unsatisfied): return 0 return 1 @@ -2241,6 +2525,7 @@ class depgraph(object): args = revised_greedy_args del revised_greedy_args + args.extend(self._gen_reinstall_sets()) self._set_args(args) myfavorites = set(myfavorites) @@ -2248,7 +2533,8 @@ class depgraph(object): if isinstance(arg, (AtomArg, PackageArg)): myfavorites.add(arg.atom) elif isinstance(arg, SetArg): - myfavorites.add(arg.arg) + if not arg.internal: + myfavorites.add(arg.arg) myfavorites = list(myfavorites) if debug: @@ -2258,7 +2544,33 @@ class depgraph(object): self._dynamic_config._initial_arg_list = args[:] return self._resolve(myfavorites) - + + def _gen_reinstall_sets(self): + + atom_list = [] + for root, atom in self._rebuild.rebuild_list: + atom_list.append((root, '__auto_rebuild__', atom)) + for root, atom in self._rebuild.reinstall_list: + atom_list.append((root, '__auto_reinstall__', atom)) + for root, atom in self._dynamic_config._slot_abi_replace_installed: + atom_list.append((root, '__auto_slot_abi_replace_installed__', atom)) + + set_dict = {} + for root, set_name, atom in atom_list: + set_dict.setdefault((root, set_name), []).append(atom) + + for (root, set_name), atoms in set_dict.items(): + yield SetArg(arg=(SETPREFIX + set_name), + # Set reset_depth=False here, since we don't want these + # special sets to interact with depth calculations (see + # the emerge --deep=DEPTH option), though we want them + # to behave like normal arguments in most other respects. + pset=InternalPackageSet(initial_atoms=atoms), + force_reinstall=True, + internal=True, + reset_depth=False, + root_config=self._frozen_config.roots[root]) + def _resolve(self, myfavorites): """Given self._dynamic_config._initial_arg_list, pull in the root nodes, call self._creategraph to process theier deps and return @@ -2270,10 +2582,7 @@ class depgraph(object): pprovideddict = pkgsettings.pprovideddict virtuals = pkgsettings.getvirtuals() args = self._dynamic_config._initial_arg_list[:] - for root, atom in chain(self._rebuild.rebuild_list, - self._rebuild.reinstall_list): - args.append(AtomArg(arg=atom, atom=atom, - root_config=self._frozen_config.roots[root])) + for arg in self._expand_set_args(args, add_to_digraph=True): for atom in arg.pset.getAtoms(): self._spinner_update() @@ -2383,22 +2692,12 @@ class depgraph(object): except self._unknown_internal_error: return False, myfavorites - digraph_set = frozenset(self._dynamic_config.digraph) - - if digraph_set.intersection( - self._dynamic_config._needed_unstable_keywords) or \ - digraph_set.intersection( - self._dynamic_config._needed_p_mask_changes) or \ - digraph_set.intersection( - self._dynamic_config._needed_use_config_changes) or \ - digraph_set.intersection( - self._dynamic_config._needed_license_changes) : - #We failed if the user needs to change the configuration - self._dynamic_config._success_without_autounmask = True + if (self._dynamic_config._slot_collision_info and + not self._accept_blocker_conflicts()) or \ + (self._dynamic_config._allow_backtracking and + "slot conflict" in self._dynamic_config._backtrack_infos): return False, myfavorites - digraph_set = None - if self._rebuild.trigger_rebuilds(): backtrack_infos = self._dynamic_config._backtrack_infos config = backtrack_infos.setdefault("config", {}) @@ -2407,6 +2706,32 @@ class depgraph(object): self._dynamic_config._need_restart = True return False, myfavorites + if "config" in self._dynamic_config._backtrack_infos and \ + ("slot_abi_mask_built" in self._dynamic_config._backtrack_infos["config"] or + "slot_abi_replace_installed" in self._dynamic_config._backtrack_infos["config"]) and \ + self.need_restart(): + return False, myfavorites + + # Any failures except those due to autounmask *alone* should return + # before this point, since the success_without_autounmask flag that's + # set below is reserved for cases where there are *zero* other + # problems. For reference, see backtrack_depgraph, where it skips the + # get_best_run() call when success_without_autounmask is True. + + digraph_nodes = self._dynamic_config.digraph.nodes + + if any(x in digraph_nodes for x in + self._dynamic_config._needed_unstable_keywords) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_p_mask_changes) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_use_config_changes) or \ + any(x in digraph_nodes for x in + self._dynamic_config._needed_license_changes) : + #We failed if the user needs to change the configuration + self._dynamic_config._success_without_autounmask = True + return False, myfavorites + # We're true here unless we are missing binaries. return (True, myfavorites) @@ -2561,6 +2886,22 @@ class depgraph(object): """This will raise InvalidDependString if necessary. If trees is None then self._dynamic_config._filtered_trees is used.""" + if not isinstance(depstring, list): + eapi = None + is_valid_flag = None + if parent is not None: + eapi = parent.metadata['EAPI'] + if not parent.installed: + is_valid_flag = parent.iuse.is_valid_flag + depstring = portage.dep.use_reduce(depstring, + uselist=myuse, opconvert=True, token_class=Atom, + is_valid_flag=is_valid_flag, eapi=eapi) + + if (self._dynamic_config.myparams.get( + "ignore_built_slot_abi_deps", "n") == "y" and + parent and parent.built): + ignore_built_slot_abi_deps(depstring) + pkgsettings = self._frozen_config.pkgsettings[root] if trees is None: trees = self._dynamic_config._filtered_trees @@ -2924,7 +3265,7 @@ class depgraph(object): def _show_unsatisfied_dep(self, root, atom, myparent=None, arg=None, - check_backtrack=False, check_autounmask_breakage=False): + check_backtrack=False, check_autounmask_breakage=False, show_req_use=None): """ When check_backtrack=True, no output is produced and the method either returns or raises _backtrack_mask if @@ -3076,7 +3417,7 @@ class depgraph(object): untouchable_flags = \ frozenset(chain(pkg.use.mask, pkg.use.force)) - if untouchable_flags.intersection( + if any(x in untouchable_flags for x in chain(need_enable, need_disable)): continue @@ -3126,7 +3467,7 @@ class depgraph(object): untouchable_flags = \ frozenset(chain(myparent.use.mask, myparent.use.force)) - if untouchable_flags.intersection(involved_flags): + if any(x in untouchable_flags for x in involved_flags): continue required_use = myparent.metadata.get("REQUIRED_USE") @@ -3205,62 +3546,66 @@ class depgraph(object): mask_docs = False - if required_use_unsatisfied: + if show_req_use is None and required_use_unsatisfied: # We have an unmasked package that only requires USE adjustment # in order to satisfy REQUIRED_USE, and nothing more. We assume # that the user wants the latest version, so only the first # instance is displayed. - pkg = required_use_unsatisfied[0] + show_req_use = required_use_unsatisfied[0] + + if show_req_use is not None: + + pkg = show_req_use output_cpv = pkg.cpv + _repo_separator + pkg.repo - writemsg_stdout("\n!!! " + \ + writemsg("\n!!! " + \ colorize("BAD", "The ebuild selected to satisfy ") + \ colorize("INFORM", xinfo) + \ colorize("BAD", " has unmet requirements.") + "\n", noiselevel=-1) use_display = pkg_use_display(pkg, self._frozen_config.myopts) - writemsg_stdout("- %s %s\n" % (output_cpv, use_display), + writemsg("- %s %s\n" % (output_cpv, use_display), noiselevel=-1) - writemsg_stdout("\n The following REQUIRED_USE flag constraints " + \ + writemsg("\n The following REQUIRED_USE flag constraints " + \ "are unsatisfied:\n", noiselevel=-1) reduced_noise = check_required_use( pkg.metadata["REQUIRED_USE"], self._pkg_use_enabled(pkg), pkg.iuse.is_valid_flag).tounicode() - writemsg_stdout(" %s\n" % \ + writemsg(" %s\n" % \ human_readable_required_use(reduced_noise), noiselevel=-1) normalized_required_use = \ " ".join(pkg.metadata["REQUIRED_USE"].split()) if reduced_noise != normalized_required_use: - writemsg_stdout("\n The above constraints " + \ + writemsg("\n The above constraints " + \ "are a subset of the following complete expression:\n", noiselevel=-1) - writemsg_stdout(" %s\n" % \ + writemsg(" %s\n" % \ human_readable_required_use(normalized_required_use), noiselevel=-1) - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n", noiselevel=-1) elif show_missing_use: - writemsg_stdout("\nemerge: there are no ebuilds built with USE flags to satisfy "+green(xinfo)+".\n", noiselevel=-1) - writemsg_stdout("!!! One of the following packages is required to complete your request:\n", noiselevel=-1) + writemsg("\nemerge: there are no ebuilds built with USE flags to satisfy "+green(xinfo)+".\n", noiselevel=-1) + writemsg("!!! One of the following packages is required to complete your request:\n", noiselevel=-1) for pkg, mreasons in show_missing_use: - writemsg_stdout("- "+pkg.cpv+_repo_separator+pkg.repo+" ("+", ".join(mreasons)+")\n", noiselevel=-1) + writemsg("- "+pkg.cpv+_repo_separator+pkg.repo+" ("+", ".join(mreasons)+")\n", noiselevel=-1) elif masked_packages: - writemsg_stdout("\n!!! " + \ + writemsg("\n!!! " + \ colorize("BAD", "All ebuilds that could satisfy ") + \ colorize("INFORM", xinfo) + \ colorize("BAD", " have been masked.") + "\n", noiselevel=-1) - writemsg_stdout("!!! One of the following masked packages is required to complete your request:\n", noiselevel=-1) + writemsg("!!! One of the following masked packages is required to complete your request:\n", noiselevel=-1) have_eapi_mask = show_masked_packages(masked_packages) if have_eapi_mask: - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n", noiselevel=-1) msg = ("The current version of portage supports " + \ "EAPI '%s'. You must upgrade to a newer version" + \ " of portage before EAPI masked packages can" + \ " be installed.") % portage.const.EAPI - writemsg_stdout("\n".join(textwrap.wrap(msg, 75)), noiselevel=-1) - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n".join(textwrap.wrap(msg, 75)), noiselevel=-1) + writemsg("\n", noiselevel=-1) mask_docs = True else: cp_exists = False @@ -3270,7 +3615,7 @@ class depgraph(object): cp_exists = True break - writemsg_stdout("\nemerge: there are no ebuilds to satisfy "+green(xinfo)+".\n", noiselevel=-1) + writemsg("\nemerge: there are no ebuilds to satisfy "+green(xinfo)+".\n", noiselevel=-1) if isinstance(myparent, AtomArg) and \ not cp_exists and \ self._frozen_config.myopts.get( @@ -3280,7 +3625,7 @@ class depgraph(object): if cat == "null": cat = None - writemsg_stdout("\nemerge: searching for similar names..." + writemsg("\nemerge: searching for similar names..." , noiselevel=-1) all_cp = set() @@ -3328,16 +3673,16 @@ class depgraph(object): matches = matches_orig_case if len(matches) == 1: - writemsg_stdout("\nemerge: Maybe you meant " + matches[0] + "?\n" + writemsg("\nemerge: Maybe you meant " + matches[0] + "?\n" , noiselevel=-1) elif len(matches) > 1: - writemsg_stdout( + writemsg( "\nemerge: Maybe you meant any of these: %s?\n" % \ (", ".join(matches),), noiselevel=-1) else: # Generally, this would only happen if # all dbapis are empty. - writemsg_stdout(" nothing similar found.\n" + writemsg(" nothing similar found.\n" , noiselevel=-1) msg = [] if not isinstance(myparent, AtomArg): @@ -3350,12 +3695,12 @@ class depgraph(object): (node)), node_type)) if msg: - writemsg_stdout("\n".join(msg), noiselevel=-1) - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n".join(msg), noiselevel=-1) + writemsg("\n", noiselevel=-1) if mask_docs: show_mask_docs() - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n", noiselevel=-1) def _iter_match_pkgs_any(self, root_config, atom, onlydeps=False): for db, pkg_type, built, installed, db_keys in \ @@ -3373,60 +3718,12 @@ class depgraph(object): """ db = root_config.trees[self.pkg_tree_map[pkg_type]].dbapi - - if hasattr(db, "xmatch"): - # For portdbapi we match only against the cpv, in order - # to bypass unnecessary cache access for things like IUSE - # and SLOT. Later, we cache the metadata in a Package - # instance, and use that for further matching. This - # optimization is especially relevant since - # pordbapi.aux_get() does not cache calls that have - # myrepo or mytree arguments. - cpv_list = db.xmatch("match-all-cpv-only", atom) - else: - cpv_list = db.match(atom) - - # USE=multislot can make an installed package appear as if - # it doesn't satisfy a slot dependency. Rebuilding the ebuild - # won't do any good as long as USE=multislot is enabled since - # the newly built package still won't have the expected slot. - # Therefore, assume that such SLOT dependencies are already - # satisfied rather than forcing a rebuild. + atom_exp = dep_expand(atom, mydb=db, settings=root_config.settings) + cp_list = db.cp_list(atom_exp.cp) + matched_something = False installed = pkg_type == 'installed' - if installed and not cpv_list and atom.slot: - if "remove" in self._dynamic_config.myparams: - # We need to search the portdbapi, which is not in our - # normal dbs list, in order to find the real SLOT. - portdb = self._frozen_config.trees[root_config.root]["porttree"].dbapi - db_keys = list(portdb._aux_cache_keys) - dbs = [(portdb, "ebuild", False, False, db_keys)] - else: - dbs = self._dynamic_config._filtered_trees[root_config.root]["dbs"] - - for cpv in db.match(atom.cp): - slot_available = False - for other_db, other_type, other_built, \ - other_installed, other_keys in dbs: - try: - if atom.slot == \ - other_db.aux_get(cpv, ["SLOT"])[0]: - slot_available = True - break - except KeyError: - pass - if not slot_available: - continue - inst_pkg = self._pkg(cpv, "installed", - root_config, installed=installed, myrepo = atom.repo) - # Remove the slot from the atom and verify that - # the package matches the resulting atom. - if portage.match_from_list( - atom.without_slot, [inst_pkg]): - yield inst_pkg - return - - if cpv_list: + if cp_list: atom_set = InternalPackageSet(initial_atoms=(atom,), allow_repo=True) if atom.repo is None and hasattr(db, "getRepositories"): @@ -3435,8 +3732,13 @@ class depgraph(object): repo_list = [atom.repo] # descending order - cpv_list.reverse() - for cpv in cpv_list: + cp_list.reverse() + for cpv in cp_list: + # Call match_from_list on one cpv at a time, in order + # to avoid unnecessary match_from_list comparisons on + # versions that are never yielded from this method. + if not match_from_list(atom_exp, [cpv]): + continue for repo in repo_list: try: @@ -3453,26 +3755,65 @@ class depgraph(object): # Make sure that cpv from the current repo satisfies the atom. # This might not be the case if there are several repos with # the same cpv, but different metadata keys, like SLOT. - # Also, for portdbapi, parts of the match that require - # metadata access are deferred until we have cached the - # metadata in a Package instance. + # Also, parts of the match that require metadata access + # are deferred until we have cached the metadata in a + # Package instance. if not atom_set.findAtomForPackage(pkg, modified_use=self._pkg_use_enabled(pkg)): continue + matched_something = True yield pkg + # USE=multislot can make an installed package appear as if + # it doesn't satisfy a slot dependency. Rebuilding the ebuild + # won't do any good as long as USE=multislot is enabled since + # the newly built package still won't have the expected slot. + # Therefore, assume that such SLOT dependencies are already + # satisfied rather than forcing a rebuild. + if not matched_something and installed and atom.slot is not None: + + if "remove" in self._dynamic_config.myparams: + # We need to search the portdbapi, which is not in our + # normal dbs list, in order to find the real SLOT. + portdb = self._frozen_config.trees[root_config.root]["porttree"].dbapi + db_keys = list(portdb._aux_cache_keys) + dbs = [(portdb, "ebuild", False, False, db_keys)] + else: + dbs = self._dynamic_config._filtered_trees[root_config.root]["dbs"] + + cp_list = db.cp_list(atom_exp.cp) + if cp_list: + atom_set = InternalPackageSet( + initial_atoms=(atom.without_slot,), allow_repo=True) + atom_exp_without_slot = atom_exp.without_slot + cp_list.reverse() + for cpv in cp_list: + if not match_from_list(atom_exp_without_slot, [cpv]): + continue + slot_available = False + for other_db, other_type, other_built, \ + other_installed, other_keys in dbs: + try: + if atom.slot == \ + other_db.aux_get(cpv, ["SLOT"])[0]: + slot_available = True + break + except KeyError: + pass + if not slot_available: + continue + inst_pkg = self._pkg(cpv, "installed", + root_config, installed=installed, myrepo=atom.repo) + # Remove the slot from the atom and verify that + # the package matches the resulting atom. + if atom_set.findAtomForPackage(inst_pkg): + yield inst_pkg + return + def _select_pkg_highest_available(self, root, atom, onlydeps=False): - cache_key = (root, atom, atom.unevaluated_atom, onlydeps) + cache_key = (root, atom, atom.unevaluated_atom, onlydeps, self._dynamic_config._autounmask) ret = self._dynamic_config._highest_pkg_cache.get(cache_key) if ret is not None: - pkg, existing = ret - if pkg and not existing: - existing = self._dynamic_config._slot_pkg_map[root].get(pkg.slot_atom) - if existing and existing == pkg: - # Update the cache to reflect that the - # package has been added to the graph. - ret = pkg, pkg - self._dynamic_config._highest_pkg_cache[cache_key] = ret return ret ret = self._select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps) self._dynamic_config._highest_pkg_cache[cache_key] = ret @@ -3489,21 +3830,55 @@ class depgraph(object): True if the user has not explicitly requested for this package to be replaced (typically via an atom on the command line). """ - if "selective" not in self._dynamic_config.myparams and \ - pkg.root == self._frozen_config.target_root: - if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, - modified_use=self._pkg_use_enabled(pkg)): - return True - try: - next(self._iter_atoms_for_pkg(pkg)) - except StopIteration: - pass - except portage.exception.InvalidDependString: - pass - else: + if self._frozen_config.excluded_pkgs.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + return True + + arg = False + try: + for arg, atom in self._iter_atoms_for_pkg(pkg): + if arg.force_reinstall: + return False + except InvalidDependString: + pass + + if "selective" in self._dynamic_config.myparams: + return True + + return not arg + + def _equiv_ebuild_visible(self, pkg, autounmask_level=None): + try: + pkg_eb = self._pkg( + pkg.cpv, "ebuild", pkg.root_config, myrepo=pkg.repo) + except portage.exception.PackageNotFound: + pkg_eb_visible = False + for pkg_eb in self._iter_match_pkgs(pkg.root_config, + "ebuild", Atom("=%s" % (pkg.cpv,))): + if self._pkg_visibility_check(pkg_eb, autounmask_level): + pkg_eb_visible = True + break + if not pkg_eb_visible: return False + else: + if not self._pkg_visibility_check(pkg_eb, autounmask_level): + return False + return True + def _equiv_binary_installed(self, pkg): + build_time = pkg.metadata.get('BUILD_TIME') + if not build_time: + return False + + try: + inst_pkg = self._pkg(pkg.cpv, "installed", + pkg.root_config, installed=True) + except PackageNotFound: + return False + + return build_time == inst_pkg.metadata.get('BUILD_TIME') + class _AutounmaskLevel(object): __slots__ = ("allow_use_changes", "allow_unstable_keywords", "allow_license_changes", \ "allow_missing_keywords", "allow_unmasks") @@ -3519,7 +3894,8 @@ class depgraph(object): """ Iterate over the different allowed things to unmask. - 1. USE + 0. USE + 1. USE + license 2. USE + ~arch + license 3. USE + ~arch + license + missing keywords 4. USE + ~arch + license + masks @@ -3538,8 +3914,12 @@ class depgraph(object): autounmask_level = self._AutounmaskLevel() autounmask_level.allow_use_changes = True + yield autounmask_level - for only_use_changes in (True, False): + autounmask_level.allow_license_changes = True + yield autounmask_level + + for only_use_changes in (False,): autounmask_level.allow_unstable_keywords = (not only_use_changes) autounmask_level.allow_license_changes = (not only_use_changes) @@ -3743,7 +4123,7 @@ class depgraph(object): new_use, changes = self._dynamic_config._needed_use_config_changes.get(pkg) for ppkg, atom in parent_atoms: if not atom.use or \ - not atom.use.required.intersection(changes): + not any(x in atom.use.required for x in changes): continue else: return True @@ -3757,8 +4137,8 @@ class depgraph(object): not check_required_use(required_use, new_use, pkg.iuse.is_valid_flag): return old_use - if pkg.use.mask.intersection(new_changes) or \ - pkg.use.force.intersection(new_changes): + if any(x in pkg.use.mask for x in new_changes) or \ + any(x in pkg.use.force for x in new_changes): return old_use self._dynamic_config._needed_use_config_changes[pkg] = (new_use, new_changes) @@ -3929,37 +4309,24 @@ class depgraph(object): if not use_ebuild_visibility and (usepkgonly or useoldpkg): if pkg.installed and pkg.masks: continue - else: - try: - pkg_eb = self._pkg( - pkg.cpv, "ebuild", root_config, myrepo=pkg.repo) - except portage.exception.PackageNotFound: - pkg_eb_visible = False - for pkg_eb in self._iter_match_pkgs(pkg.root_config, - "ebuild", Atom("=%s" % (pkg.cpv,))): - if self._pkg_visibility_check(pkg_eb, autounmask_level): - pkg_eb_visible = True - break - if not pkg_eb_visible: - continue - else: - if not self._pkg_visibility_check(pkg_eb, autounmask_level): - continue + elif not self._equiv_ebuild_visible(pkg, + autounmask_level=autounmask_level): + continue # Calculation of USE for unbuilt ebuilds is relatively # expensive, so it is only performed lazily, after the # above visibility checks are complete. myarg = None - if root == self._frozen_config.target_root: - try: - myarg = next(self._iter_atoms_for_pkg(pkg)) - except StopIteration: - pass - except portage.exception.InvalidDependString: - if not installed: - # masked by corruption - continue + try: + for myarg, myarg_atom in self._iter_atoms_for_pkg(pkg): + if myarg.force_reinstall: + reinstall = True + break + except InvalidDependString: + if not installed: + # masked by corruption + continue if not installed and myarg: found_available_arg = True @@ -3989,7 +4356,7 @@ class depgraph(object): missing_disabled = atom.use.missing_disabled.difference(pkg.iuse.all) if atom.use.enabled: - if atom.use.enabled.intersection(missing_disabled): + if any(x in atom.use.enabled for x in missing_disabled): use_match = False can_adjust_use = False need_enabled = atom.use.enabled.difference(use) @@ -3998,11 +4365,11 @@ class depgraph(object): if need_enabled: use_match = False if can_adjust_use: - if pkg.use.mask.intersection(need_enabled): + if any(x in pkg.use.mask for x in need_enabled): can_adjust_use = False if atom.use.disabled: - if atom.use.disabled.intersection(missing_enabled): + if any(x in atom.use.disabled for x in missing_enabled): use_match = False can_adjust_use = False need_disabled = atom.use.disabled.intersection(use) @@ -4011,8 +4378,8 @@ class depgraph(object): if need_disabled: use_match = False if can_adjust_use: - if pkg.use.force.difference( - pkg.use.mask).intersection(need_disabled): + if any(x in pkg.use.force and x not in + pkg.use.mask for x in need_disabled): can_adjust_use = False if not use_match: @@ -4278,9 +4645,19 @@ class depgraph(object): "recurse" not in self._dynamic_config.myparams: return 1 + complete_if_new_use = self._dynamic_config.myparams.get( + "complete_if_new_use", "y") == "y" + complete_if_new_ver = self._dynamic_config.myparams.get( + "complete_if_new_ver", "y") == "y" + rebuild_if_new_slot_abi = self._dynamic_config.myparams.get( + "rebuild_if_new_slot_abi", "y") == "y" + complete_if_new_slot = rebuild_if_new_slot_abi + if "complete" not in self._dynamic_config.myparams and \ - self._dynamic_config.myparams.get("complete_if_new_ver", "y") == "y": - # Enable complete mode if an installed package version will change. + (complete_if_new_use or + complete_if_new_ver or complete_if_new_slot): + # Enable complete mode if an installed package will change somehow. + use_change = False version_change = False for node in self._dynamic_config.digraph: if not isinstance(node, Package) or \ @@ -4288,12 +4665,35 @@ class depgraph(object): continue vardb = self._frozen_config.roots[ node.root].trees["vartree"].dbapi - inst_pkg = vardb.match_pkgs(node.slot_atom) - if inst_pkg and (inst_pkg[0] > node or inst_pkg[0] < node): - version_change = True - break - if version_change: + if complete_if_new_use or complete_if_new_ver: + inst_pkg = vardb.match_pkgs(node.slot_atom) + if inst_pkg and inst_pkg[0].cp == node.cp: + inst_pkg = inst_pkg[0] + if complete_if_new_ver and \ + (inst_pkg < node or node < inst_pkg): + version_change = True + break + + # Intersect enabled USE with IUSE, in order to + # ignore forced USE from implicit IUSE flags, since + # they're probably irrelevant and they are sensitive + # to use.mask/force changes in the profile. + if complete_if_new_use and \ + (node.iuse.all != inst_pkg.iuse.all or + self._pkg_use_enabled(node).intersection(node.iuse.all) != + self._pkg_use_enabled(inst_pkg).intersection(inst_pkg.iuse.all)): + use_change = True + break + + if complete_if_new_slot: + cp_list = vardb.match_pkgs(Atom(node.cp)) + if (cp_list and cp_list[0].cp == node.cp and + not any(node.slot == pkg.slot for pkg in cp_list)): + version_change = True + break + + if use_change or version_change: self._dynamic_config.myparams["complete"] = True if "complete" not in self._dynamic_config.myparams: @@ -4307,6 +4707,7 @@ class depgraph(object): # scheduled for replacement. Also, toggle the "deep" # parameter so that all dependencies are traversed and # accounted for. + self._dynamic_config._complete_mode = True self._select_atoms = self._select_atoms_from_graph if "remove" in self._dynamic_config.myparams: self._select_package = self._select_pkg_from_installed @@ -4325,7 +4726,8 @@ class depgraph(object): args = self._dynamic_config._initial_arg_list[:] for root in self._frozen_config.roots: if root != self._frozen_config.target_root and \ - "remove" in self._dynamic_config.myparams: + ("remove" in self._dynamic_config.myparams or + self._frozen_config.myopts.get("--root-deps") is not None): # Only pull in deps for the relevant root. continue depgraph_sets = self._dynamic_config.sets[root] @@ -4466,6 +4868,11 @@ class depgraph(object): # are already built. dep_keys = ["RDEPEND", "PDEPEND"] for myroot in self._frozen_config.trees: + + if self._frozen_config.myopts.get("--root-deps") is not None and \ + myroot != self._frozen_config.target_root: + continue + vardb = self._frozen_config.trees[myroot]["vartree"].dbapi pkgsettings = self._frozen_config.pkgsettings[myroot] root_config = self._frozen_config.roots[myroot] @@ -4916,16 +5323,24 @@ class depgraph(object): root_config.root]["root_config"] = root_config def _resolve_conflicts(self): + + if "complete" not in self._dynamic_config.myparams and \ + self._dynamic_config._allow_backtracking and \ + self._dynamic_config._slot_collision_nodes and \ + not self._accept_blocker_conflicts(): + self._dynamic_config.myparams["complete"] = True + if not self._complete_graph(): raise self._unknown_internal_error() + self._process_slot_conflicts() + + self._slot_abi_trigger_reinstalls() + if not self._validate_blockers(): self._dynamic_config._skip_restart = True raise self._unknown_internal_error() - if self._dynamic_config._slot_collision_info: - self._process_slot_conflicts() - def _serialize_tasks(self): debug = "--debug" in self._frozen_config.myopts @@ -5220,7 +5635,7 @@ class depgraph(object): for node in nodes: parents = mygraph.parent_nodes(node, ignore_priority=DepPrioritySatisfiedRange.ignore_soft) - if parents and set(parents).intersection(asap_nodes): + if any(x in asap_nodes for x in parents): selected_nodes = [node] break else: @@ -6094,27 +6509,27 @@ class depgraph(object): settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH) if len(roots) > 1: - writemsg_stdout("\nFor %s:\n" % abs_user_config, noiselevel=-1) + writemsg("\nFor %s:\n" % abs_user_config, noiselevel=-1) if root in unstable_keyword_msg: - writemsg_stdout("\nThe following " + colorize("BAD", "keyword changes") + \ + writemsg("\nThe following " + colorize("BAD", "keyword changes") + \ " are necessary to proceed:\n", noiselevel=-1) - writemsg_stdout(format_msg(unstable_keyword_msg[root]), noiselevel=-1) + writemsg(format_msg(unstable_keyword_msg[root]), noiselevel=-1) if root in p_mask_change_msg: - writemsg_stdout("\nThe following " + colorize("BAD", "mask changes") + \ + writemsg("\nThe following " + colorize("BAD", "mask changes") + \ " are necessary to proceed:\n", noiselevel=-1) - writemsg_stdout(format_msg(p_mask_change_msg[root]), noiselevel=-1) + writemsg(format_msg(p_mask_change_msg[root]), noiselevel=-1) if root in use_changes_msg: - writemsg_stdout("\nThe following " + colorize("BAD", "USE changes") + \ + writemsg("\nThe following " + colorize("BAD", "USE changes") + \ " are necessary to proceed:\n", noiselevel=-1) - writemsg_stdout(format_msg(use_changes_msg[root]), noiselevel=-1) + writemsg(format_msg(use_changes_msg[root]), noiselevel=-1) if root in license_msg: - writemsg_stdout("\nThe following " + colorize("BAD", "license changes") + \ + writemsg("\nThe following " + colorize("BAD", "license changes") + \ " are necessary to proceed:\n", noiselevel=-1) - writemsg_stdout(format_msg(license_msg[root]), noiselevel=-1) + writemsg(format_msg(license_msg[root]), noiselevel=-1) protect_obj = {} if write_to_file: @@ -6161,7 +6576,7 @@ class depgraph(object): for line in msg: if line: line = colorize("INFORM", line) - writemsg_stdout(line + "\n", noiselevel=-1) + writemsg(line + "\n", noiselevel=-1) if ask and write_to_file and file_to_write_to: prompt = "\nWould you like to add these " + \ @@ -6193,14 +6608,14 @@ class depgraph(object): file_to_write_to.get((abs_user_config, "package.license"))) if problems: - writemsg_stdout("\nThe following problems occurred while writing autounmask changes:\n", \ + writemsg("\nThe following problems occurred while writing autounmask changes:\n", \ noiselevel=-1) - writemsg_stdout("".join(problems), noiselevel=-1) + writemsg("".join(problems), noiselevel=-1) elif write_to_file and roots: - writemsg_stdout("\nAutounmask changes successfully written. Remember to run dispatch-conf.\n", \ + writemsg("\nAutounmask changes successfully written. Remember to run dispatch-conf.\n", \ noiselevel=-1) elif not pretend and not autounmask_write and roots: - writemsg_stdout("\nUse --autounmask-write to write changes to config files (honoring CONFIG_PROTECT).\n", \ + writemsg("\nUse --autounmask-write to write changes to config files (honoring CONFIG_PROTECT).\n", \ noiselevel=-1) @@ -6211,35 +6626,8 @@ class depgraph(object): the merge list where it is most likely to be seen, but if display() is not going to be called then this method should be called explicitly to ensure that the user is notified of problems with the graph. - - All output goes to stderr, except for unsatisfied dependencies which - go to stdout for parsing by programs such as autounmask. """ - # Note that show_masked_packages() sends its output to - # stdout, and some programs such as autounmask parse the - # output in cases when emerge bails out. However, when - # show_masked_packages() is called for installed packages - # here, the message is a warning that is more appropriate - # to send to stderr, so temporarily redirect stdout to - # stderr. TODO: Fix output code so there's a cleaner way - # to redirect everything to stderr. - sys.stdout.flush() - sys.stderr.flush() - stdout = sys.stdout - try: - sys.stdout = sys.stderr - self._display_problems() - finally: - sys.stdout = stdout - sys.stdout.flush() - sys.stderr.flush() - - # This goes to stdout for parsing by programs like autounmask. - for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: - self._show_unsatisfied_dep(*pargs, **kwargs) - - def _display_problems(self): if self._dynamic_config._circular_deps_for_display is not None: self._show_circular_deps( self._dynamic_config._circular_deps_for_display) @@ -6358,6 +6746,9 @@ class depgraph(object): show_mask_docs() writemsg("\n", noiselevel=-1) + for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: + self._show_unsatisfied_dep(*pargs, **kwargs) + def saveNomergeFavorites(self): """Find atoms in favorites that are not in the mergelist and add them to the world file if necessary.""" @@ -6404,6 +6795,9 @@ class depgraph(object): continue if arg.root_config.root != root_config.root: continue + if arg.internal: + # __auto_* sets + continue k = arg.name if k in ("selected", "world") or \ not root_config.sets[k].world_candidate: @@ -6415,9 +6809,13 @@ class depgraph(object): all_added.extend(added_favorites) all_added.sort() for a in all_added: + if a.startswith(SETPREFIX): + filename = "world_sets" + else: + filename = "world" writemsg_stdout( - ">>> Recording %s in \"world\" favorites file...\n" % \ - colorize("INFORM", str(a)), noiselevel=-1) + ">>> Recording %s in \"%s\" favorites file...\n" % + (colorize("INFORM", _unicode(a)), filename), noiselevel=-1) if all_added: world_set.update(all_added) @@ -6743,7 +7141,8 @@ class _dep_check_composite_db(dbapi): return ret def match(self, atom): - ret = self._match_cache.get(atom) + cache_key = (atom, atom.unevaluated_atom) + ret = self._match_cache.get(cache_key) if ret is not None: return ret[:] @@ -6791,17 +7190,12 @@ class _dep_check_composite_db(dbapi): if len(ret) > 1: self._cpv_sort_ascending(ret) - self._match_cache[atom] = ret + self._match_cache[cache_key] = ret return ret[:] def _visible(self, pkg): - if pkg.installed and "selective" not in self._depgraph._dynamic_config.myparams: - try: - arg = next(self._depgraph._iter_atoms_for_pkg(pkg)) - except (StopIteration, portage.exception.InvalidDependString): - arg = None - if arg: - return False + if pkg.installed and not self._depgraph._want_installed_pkg(pkg): + return False if pkg.installed and \ (pkg.masks or not self._depgraph._pkg_visibility_check(pkg)): # Account for packages with masks (like KEYWORDS masks) @@ -6817,24 +7211,8 @@ class _dep_check_composite_db(dbapi): if not avoid_update: if not use_ebuild_visibility and usepkgonly: return False - else: - try: - pkg_eb = self._depgraph._pkg( - pkg.cpv, "ebuild", pkg.root_config, - myrepo=pkg.repo) - except portage.exception.PackageNotFound: - pkg_eb_visible = False - for pkg_eb in self._depgraph._iter_match_pkgs( - pkg.root_config, "ebuild", - Atom("=%s" % (pkg.cpv,))): - if self._depgraph._pkg_visibility_check(pkg_eb): - pkg_eb_visible = True - break - if not pkg_eb_visible: - return False - else: - if not self._depgraph._pkg_visibility_check(pkg_eb): - return False + elif not self._depgraph._equiv_ebuild_visible(pkg): + return False in_graph = self._depgraph._dynamic_config._slot_pkg_map[ self._root].get(pkg.slot_atom) @@ -6847,7 +7225,7 @@ class _dep_check_composite_db(dbapi): # Note: highest_visible is not necessarily the real highest # visible, especially when --update is not enabled, so use # < operator instead of !=. - if pkg < highest_visible: + if highest_visible is not None and pkg < highest_visible: return False elif in_graph != pkg: # Mask choices for packages that would trigger a slot @@ -7114,8 +7492,6 @@ def get_mask_info(root_config, cpv, pkgsettings, mreasons = ["corruption"] else: eapi = metadata['EAPI'] - if eapi[:1] == '-': - eapi = eapi[1:] if not portage.eapi_is_supported(eapi): mreasons = ['EAPI %s' % eapi] else: @@ -7172,10 +7548,11 @@ def show_masked_packages(masked_packages): # above via mreasons. pass - writemsg_stdout("- "+output_cpv+" (masked by: "+", ".join(mreasons)+")\n", noiselevel=-1) + writemsg("- "+output_cpv+" (masked by: "+", ".join(mreasons)+")\n", + noiselevel=-1) if comment and comment not in shown_comments: - writemsg_stdout(filename + ":\n" + comment + "\n", + writemsg(filename + ":\n" + comment + "\n", noiselevel=-1) shown_comments.add(comment) portdb = root_config.trees["porttree"].dbapi @@ -7185,13 +7562,14 @@ def show_masked_packages(masked_packages): continue msg = ("A copy of the '%s' license" + \ " is located at '%s'.\n\n") % (l, l_path) - writemsg_stdout(msg, noiselevel=-1) + writemsg(msg, noiselevel=-1) shown_licenses.add(l) return have_eapi_mask def show_mask_docs(): - writemsg_stdout("For more information, see the MASKED PACKAGES section in the emerge\n", noiselevel=-1) - writemsg_stdout("man page or refer to the Gentoo Handbook.\n", noiselevel=-1) + writemsg("For more information, see the MASKED PACKAGES " + "section in the emerge\n", noiselevel=-1) + writemsg("man page or refer to the Gentoo Handbook.\n", noiselevel=-1) def show_blocker_docs_link(): writemsg("\nFor more information about " + bad("Blocked Packages") + ", please refer to the following\n", noiselevel=-1) diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 0fbc4b7f725..f19994c46c7 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -13,6 +13,7 @@ import platform import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.news:count_unread_news,display_news_notifications', + 'portage.emaint.modules.logs.logs:CleanLogs', ) from portage import os from portage import _encodings @@ -381,7 +382,9 @@ def post_emerge(myaction, myopts, myfiles, _flush_elog_mod_echo() if not vardbapi._pkgs_changed: - display_news_notification(root_config, myopts) + # GLEP 42 says to display news *after* an emerge --pretend + if "--pretend" in myopts: + display_news_notification(root_config, myopts) # If vdb state has not changed then there's nothing else to do. return @@ -402,11 +405,10 @@ def post_emerge(myaction, myopts, myfiles, if vdb_lock: vardbapi.unlock() + display_preserved_libs(vardbapi, myopts) chk_updated_cfg_files(settings['EROOT'], config_protect) display_news_notification(root_config, myopts) - if retval in (None, os.EX_OK) or (not "--pretend" in myopts): - display_preserved_libs(vardbapi, myopts) postemerge = os.path.join(settings["PORTAGE_CONFIGROOT"], portage.USER_CONFIG_PATH, "bin", "post_emerge") @@ -476,6 +478,7 @@ def insert_optional_args(args): '--package-moves' : y_or_n, '--quiet' : y_or_n, '--quiet-build' : y_or_n, + '--rebuild-if-new-slot-abi': y_or_n, '--rebuild-if-new-rev' : y_or_n, '--rebuild-if-new-ver' : y_or_n, '--rebuild-if-unbuilt' : y_or_n, @@ -701,6 +704,12 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n }, + "--complete-graph-if-new-use": { + "help" : "trigger --complete-graph behavior if USE or IUSE will change for an installed package", + "type" : "choice", + "choices" : y_or_n + }, + "--complete-graph-if-new-ver": { "help" : "trigger --complete-graph behavior if an installed package version will change (upgrade or downgrade)", "type" : "choice", @@ -745,6 +754,16 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n }, + "--ignore-built-slot-abi-deps": { + "help": "Ignore the SLOT/ABI := operator parts of dependencies that have " + "been recorded when packages where built. This option is intended " + "only for debugging purposes, and it only affects built packages " + "that specify SLOT/ABI := operator dependencies using the " + "experimental \"4-slot-abi\" EAPI.", + "type": "choice", + "choices": y_or_n + }, + "--jobs": { "shortopt" : "-j", @@ -858,6 +877,15 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n, }, + "--rebuild-if-new-slot-abi": { + "help" : ("Automatically rebuild or reinstall packages when SLOT/ABI := " + "operator dependencies can be satisfied by a newer slot, so that " + "older packages slots will become eligible for removal by the " + "--depclean action as soon as possible."), + "type" : "choice", + "choices" : true_y_or_n + }, + "--rebuild-if-new-rev": { "help" : "Rebuild packages when dependencies that are " + \ "used at both build-time and run-time are built, " + \ @@ -1099,6 +1127,9 @@ def parse_opts(tmpcmdline, silent=False): if myoptions.quiet_build in true_y: myoptions.quiet_build = 'y' + if myoptions.rebuild_if_new_slot_abi in true_y: + myoptions.rebuild_if_new_slot_abi = 'y' + if myoptions.rebuild_if_new_ver in true_y: myoptions.rebuild_if_new_ver = True else: @@ -1321,39 +1352,22 @@ def clean_logs(settings): if "clean-logs" not in settings.features: return - clean_cmd = settings.get("PORT_LOGDIR_CLEAN") - if clean_cmd: - clean_cmd = shlex_split(clean_cmd) - if not clean_cmd: - return - logdir = settings.get("PORT_LOGDIR") if logdir is None or not os.path.isdir(logdir): return - variables = {"PORT_LOGDIR" : logdir} - cmd = [varexpand(x, mydict=variables) for x in clean_cmd] - - try: - rval = portage.process.spawn(cmd, env=os.environ) - except portage.exception.CommandNotFound: - rval = 127 - - if rval != os.EX_OK: - out = portage.output.EOutput() - out.eerror("PORT_LOGDIR_CLEAN returned %d" % (rval,)) - out.eerror("See the make.conf(5) man page for " - "PORT_LOGDIR_CLEAN usage instructions.") + options = { + 'eerror': portage.output.EOutput().eerror, + # uncomment next line to output a succeeded message + #'einfo': portage.output.EOutput().einfo + } + cleanlogs = CleanLogs() + cleanlogs.clean(settings=settings, options=options) def setconfig_fallback(root_config): - from portage._sets.base import DummyPackageSet - from portage._sets.files import WorldSelectedSet - from portage._sets.profiles import PackagesSystemSet setconfig = root_config.setconfig - setconfig.psets['world'] = DummyPackageSet(atoms=['@selected', '@system']) - setconfig.psets['selected'] = WorldSelectedSet(root_config.settings['EROOT']) - setconfig.psets['system'] = \ - PackagesSystemSet(root_config.settings.profiles) + setconfig._create_default_config() + setconfig._parse(update=True) root_config.sets = setconfig.getSets() def get_missing_sets(root_config): @@ -1475,6 +1489,12 @@ def expand_set_arguments(myfiles, myaction, root_config): writemsg_level(("emerge: the given set '%s' " + \ "contains a non-existent set named '%s'.\n") % \ (s, e), level=logging.ERROR, noiselevel=-1) + if s in ('world', 'selected') and \ + SETPREFIX + e.value in sets['selected']: + writemsg_level(("Use `emerge --deselect %s%s` to " + "remove this set from world_sets.\n") % + (SETPREFIX, e,), level=logging.ERROR, + noiselevel=-1) return (None, 1) if myaction in unmerge_actions and \ not sets[s].supportsOperation("unmerge"): @@ -2033,6 +2053,7 @@ def emerge_main(args=None): level=logging.ERROR, noiselevel=-1) return 1 + # GLEP 42 says to display news *after* an emerge --pretend if "--pretend" not in myopts: display_news_notification(root_config, myopts) retval = action_build(settings, trees, mtimedb, diff --git a/pym/_emerge/resolver/backtracking.py b/pym/_emerge/resolver/backtracking.py index f2857b0f576..d8f49c679c2 100644 --- a/pym/_emerge/resolver/backtracking.py +++ b/pym/_emerge/resolver/backtracking.py @@ -7,7 +7,8 @@ class BacktrackParameter(object): __slots__ = ( "needed_unstable_keywords", "runtime_pkg_mask", "needed_use_config_changes", "needed_license_changes", - "rebuild_list", "reinstall_list", "needed_p_mask_changes" + "rebuild_list", "reinstall_list", "needed_p_mask_changes", + "slot_abi_replace_installed" ) def __init__(self): @@ -18,6 +19,7 @@ class BacktrackParameter(object): self.needed_license_changes = {} self.rebuild_list = set() self.reinstall_list = set() + self.slot_abi_replace_installed = set() def __deepcopy__(self, memo=None): if memo is None: @@ -29,11 +31,16 @@ class BacktrackParameter(object): #to our sets and dicts. The existing content is immutable. result.needed_unstable_keywords = copy.copy(self.needed_unstable_keywords) result.needed_p_mask_changes = copy.copy(self.needed_p_mask_changes) - result.runtime_pkg_mask = copy.copy(self.runtime_pkg_mask) result.needed_use_config_changes = copy.copy(self.needed_use_config_changes) result.needed_license_changes = copy.copy(self.needed_license_changes) result.rebuild_list = copy.copy(self.rebuild_list) result.reinstall_list = copy.copy(self.reinstall_list) + result.slot_abi_replace_installed = copy.copy(self.slot_abi_replace_installed) + + # runtime_pkg_mask contains nested dicts that must also be copied + result.runtime_pkg_mask = {} + for k, v in self.runtime_pkg_mask.items(): + result.runtime_pkg_mask[k] = copy.copy(v) return result @@ -44,7 +51,8 @@ class BacktrackParameter(object): self.needed_use_config_changes == other.needed_use_config_changes and \ self.needed_license_changes == other.needed_license_changes and \ self.rebuild_list == other.rebuild_list and \ - self.reinstall_list == other.reinstall_list + self.reinstall_list == other.reinstall_list and \ + self.slot_abi_replace_installed == other.slot_abi_replace_installed class _BacktrackNode(object): @@ -114,9 +122,10 @@ class Backtracker(object): before, we revert the mask for other packages (bug 375573). """ - for pkg in runtime_pkg_mask: + for pkg, mask_info in runtime_pkg_mask.items(): - if "missing dependency" in runtime_pkg_mask[pkg]: + if "missing dependency" in mask_info or \ + "slot_abi_mask_built" in mask_info: continue entry_is_valid = False @@ -131,6 +140,13 @@ class Backtracker(object): return True + def _feedback_slot_conflicts(self, conflicts_data): + # Only create BacktrackNode instances for the first + # conflict which occurred, since the conflicts that + # occurred later may have been caused by the first + # conflict. + self._feedback_slot_conflict(conflicts_data[0]) + def _feedback_slot_conflict(self, conflict_data): for pkg, parent_atoms in conflict_data: new_node = copy.deepcopy(self._current_node) @@ -174,6 +190,14 @@ class Backtracker(object): elif change == "needed_use_config_changes": for pkg, (new_use, new_changes) in data: para.needed_use_config_changes[pkg] = (new_use, new_changes) + elif change == "slot_conflict_abi": + new_node.terminal = False + elif change == "slot_abi_mask_built": + for pkg, mask_reasons in data.items(): + para.runtime_pkg_mask.setdefault(pkg, + {}).update(mask_reasons) + elif change == "slot_abi_replace_installed": + para.slot_abi_replace_installed.update(data) elif change == "rebuild_list": para.rebuild_list.update(data) elif change == "reinstall_list": @@ -196,7 +220,7 @@ class Backtracker(object): #There is at most one of the following types of conflicts for a given restart. if "slot conflict" in infos: - self._feedback_slot_conflict(infos["slot conflict"]) + self._feedback_slot_conflicts(infos["slot conflict"]) elif "missing dependency" in infos: self._feedback_missing_dep(infos["missing dependency"]) diff --git a/pym/_emerge/resolver/output.py b/pym/_emerge/resolver/output.py index 1208bf99b66..61cfe9e980d 100644 --- a/pym/_emerge/resolver/output.py +++ b/pym/_emerge/resolver/output.py @@ -516,8 +516,7 @@ class Display(object): @return addl: formatted slot info @return myoldbest: installed version list - Modifies self.counters.downgrades, self.counters.upgrades, - self.counters.binary + Modifies self.counters.downgrades, self.counters.upgrades """ addl = " " + pkg_info.fetch_symbol if not cpvequal(pkg.cpv, @@ -526,15 +525,11 @@ class Display(object): addl += turquoise("U")+blue("D") if pkg_info.ordered: self.counters.downgrades += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 else: # Update in slot addl += turquoise("U") + " " if pkg_info.ordered: self.counters.upgrades += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 return addl @@ -543,13 +538,11 @@ class Display(object): @return addl: formatted slot info @return myoldbest: installed version list - Modifies self.counters.newslot, self.counters.binary + Modifies self.counters.newslot """ addl = " " + green("NS") + pkg_info.fetch_symbol + " " if pkg_info.ordered: self.counters.newslot += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 return addl @@ -651,6 +644,12 @@ class Display(object): pkg_info.built = pkg.type_name != "ebuild" pkg_info.ebuild_path = None pkg_info.repo_name = pkg.repo + if ordered: + if pkg_info.merge: + if pkg.type_name == "binary": + self.counters.binary += 1 + elif pkg_info.operation == "uninstall": + self.counters.uninst += 1 if pkg.type_name == "ebuild": pkg_info.ebuild_path = self.portdb.findname( pkg.cpv, myrepo=pkg_info.repo_name) @@ -751,7 +750,7 @@ class Display(object): @param pkg: _emerge.Package.Package instance @param pkg_info: dictionay @rtype addl, myoldbest: list, myinslotlist: list - Modifies self.counters.reinst, self.counters.binary, self.counters.new + Modifies self.counters.reinst, self.counters.new """ myoldbest = [] @@ -765,10 +764,6 @@ class Display(object): if pkg_info.ordered: if pkg_info.merge: self.counters.reinst += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 - elif pkg_info.operation == "uninstall": - self.counters.uninst += 1 # filter out old-style virtual matches elif installed_versions and \ installed_versions[0].cp == pkg.cp: @@ -790,8 +785,6 @@ class Display(object): addl = " " + green("N") + " " + pkg_info.fetch_symbol + " " if pkg_info.ordered: self.counters.new += 1 - if pkg.type_name == "binary": - self.counters.binary += 1 return addl, myoldbest, myinslotlist diff --git a/pym/_emerge/resolver/slot_collision.py b/pym/_emerge/resolver/slot_collision.py index 1d522aabd61..783a6483dfd 100644 --- a/pym/_emerge/resolver/slot_collision.py +++ b/pym/_emerge/resolver/slot_collision.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -80,6 +80,8 @@ class slot_conflict_handler(object): the needed USE changes and prepare the message for the user. """ + _check_configuration_max = 1024 + def __init__(self, depgraph): self.depgraph = depgraph self.myopts = depgraph._frozen_config.myopts @@ -273,19 +275,29 @@ class slot_conflict_handler(object): if not atom_without_use_set.findAtomForPackage(other_pkg, \ modified_use=_pkg_use_enabled(other_pkg)): - #The version range does not match. - sub_type = None - if atom.operator in (">=", ">"): - sub_type = "ge" - elif atom.operator in ("=", "~"): - sub_type = "eq" - elif atom.operator in ("<=", "<"): - sub_type = "le" - - atoms = collision_reasons.get(("version", sub_type), set()) - atoms.add((ppkg, atom, other_pkg)) - num_all_specific_atoms += 1 - collision_reasons[("version", sub_type)] = atoms + if atom.operator is not None: + # The version range does not match. + sub_type = None + if atom.operator in (">=", ">"): + sub_type = "ge" + elif atom.operator in ("=", "~"): + sub_type = "eq" + elif atom.operator in ("<=", "<"): + sub_type = "le" + + key = ("version", sub_type) + atoms = collision_reasons.get(key, set()) + atoms.add((ppkg, atom, other_pkg)) + num_all_specific_atoms += 1 + collision_reasons[key] = atoms + else: + # The slot_abi does not match. + key = ("sub-slot", atom.slot_abi) + atoms = collision_reasons.get(key, set()) + atoms.add((ppkg, atom, other_pkg)) + num_all_specific_atoms += 1 + collision_reasons[key] = atoms + elif not atom_set.findAtomForPackage(other_pkg, \ modified_use=_pkg_use_enabled(other_pkg)): missing_iuse = other_pkg.iuse.get_missing_iuse( @@ -331,6 +343,9 @@ class slot_conflict_handler(object): else: best_matches[atom.cp] = (ppkg, atom) selected_for_display.update(best_matches.values()) + elif type == "sub-slot": + for ppkg, atom, other_pkg in parents: + selected_for_display.add((ppkg, atom)) elif type == "use": #Prefer atoms with unconditional use deps over, because it's #not possible to change them on the parent, which means there @@ -435,19 +450,22 @@ class slot_conflict_handler(object): # Display the specific atom from SetArg or # Package types. version_violated = False + sub_slot_violated = False use = [] for (type, sub_type), parents in collision_reasons.items(): for x in parents: if parent == x[0] and atom == x[1]: if type == "version": version_violated = True + elif type == "sub-slot": + sub_slot_violated = True elif type == "use": use.append(sub_type) break atom_str = highlight_violations(atom.unevaluated_atom, version_violated, use) - if version_violated: + if version_violated or sub_slot_violated: self.is_a_version_conflict = True msg.append("%s required by %s" % (atom_str, parent)) @@ -663,14 +681,24 @@ class slot_conflict_handler(object): solutions = [] sol_gen = _solution_candidate_generator(all_involved_flags) - while(True): + checked = 0 + while True: candidate = sol_gen.get_candidate() if not candidate: break solution = self._check_solution(config, candidate, all_conflict_atoms_by_slotatom) + checked += 1 if solution: solutions.append(solution) - + + if checked >= self._check_configuration_max: + # TODO: Implement early elimination for candidates that would + # change forced or masked flags, and don't count them here. + if self.debug: + writemsg("\nAborting _check_configuration due to " + "excessive number of candidates.\n", noiselevel=-1) + break + if self.debug: if not solutions: writemsg("No viable solutions. Rejecting configuration.\n", noiselevel=-1) diff --git a/pym/_emerge/unmerge.py b/pym/_emerge/unmerge.py index 89eae0864d3..b46b89cb8bc 100644 --- a/pym/_emerge/unmerge.py +++ b/pym/_emerge/unmerge.py @@ -13,7 +13,7 @@ from portage.dbapi._expand_new_virt import expand_new_virt from portage.output import bold, colorize, darkgreen, green from portage._sets import SETPREFIX from portage._sets.base import EditablePackageSet -from portage.util import cmp_sort_key +from portage.versions import cpv_sort_key, _pkg_str from _emerge.emergelog import emergelog from _emerge.Package import Package @@ -468,20 +468,22 @@ def _unmerge_display(root_config, myopts, unmerge_action, if not quiet: writemsg_level((mytype + ": ").rjust(14), noiselevel=-1) if pkgmap[x][mytype]: - sorted_pkgs = [portage.catpkgsplit(mypkg)[1:] for mypkg in pkgmap[x][mytype]] - sorted_pkgs.sort(key=cmp_sort_key(portage.pkgcmp)) - for pn, ver, rev in sorted_pkgs: - if rev == "r0": - myversion = ver - else: - myversion = ver + "-" + rev + sorted_pkgs = [] + for mypkg in pkgmap[x][mytype]: + try: + sorted_pkgs.append(mypkg.cpv) + except AttributeError: + sorted_pkgs.append(_pkg_str(mypkg)) + sorted_pkgs.sort(key=cpv_sort_key()) + for mypkg in sorted_pkgs: if mytype == "selected": writemsg_level( - colorize("UNMERGE_WARN", myversion + " "), + colorize("UNMERGE_WARN", mypkg.version + " "), noiselevel=-1) else: writemsg_level( - colorize("GOOD", myversion + " "), noiselevel=-1) + colorize("GOOD", mypkg.version + " "), + noiselevel=-1) else: writemsg_level("none ", noiselevel=-1) if not quiet: diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index 3495b96eca7..46bdc961c04 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -408,7 +408,7 @@ def abssymlink(symlink, target=None): _doebuild_manifest_exempt_depend = 0 -_testing_eapis = frozenset(["4-python"]) +_testing_eapis = frozenset(["4-python", "4-slot-abi"]) _deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1"]) def _eapi_is_deprecated(eapi): @@ -435,29 +435,25 @@ def eapi_is_supported(eapi): return False return eapi <= portage.const.EAPI -# Generally, it's best not to assume that cache entries for unsupported EAPIs -# can be validated. However, the current package manager specification does not -# guarantee that the EAPI can be parsed without sourcing the ebuild, so -# it's too costly to discard existing cache entries for unsupported EAPIs. -# Therefore, by default, assume that cache entries for unsupported EAPIs can be -# validated. If FEATURES=parse-eapi-* is enabled, this assumption is discarded -# since the EAPI can be determined without the incurring the cost of sourcing -# the ebuild. -_validate_cache_for_unsupported_eapis = True - -_parse_eapi_ebuild_head_re = re.compile(r'^EAPI=[\'"]?([^\'"#]*)') -_parse_eapi_ebuild_head_max_lines = 30 +# This pattern is specified by PMS section 7.3.1. +_pms_eapi_re = re.compile(r"^[ \t]*EAPI=(['\"]?)([A-Za-z0-9+_.-]*)\1[ \t]*([ \t]#.*)?$") +_comment_or_blank_line = re.compile(r"^\s*(#.*)?$") def _parse_eapi_ebuild_head(f): - count = 0 + eapi = None + eapi_lineno = None + lineno = 0 for line in f: - m = _parse_eapi_ebuild_head_re.match(line) - if m is not None: - return m.group(1).strip() - count += 1 - if count >= _parse_eapi_ebuild_head_max_lines: + lineno += 1 + m = _comment_or_blank_line.match(line) + if m is None: + eapi_lineno = lineno + m = _pms_eapi_re.match(line) + if m is not None: + eapi = m.group(2) break - return '0' + + return (eapi, eapi_lineno) def _movefile(src, dest, **kwargs): """Calls movefile and raises a PortageException if an error occurs.""" @@ -476,8 +472,7 @@ auxdbkeys = ( auxdbkeylen=len(auxdbkeys) def portageexit(): - if data.secpass > 1 and os.environ.get("SANDBOX_ON") != "1": - close_portdbapi_caches() + close_portdbapi_caches() class _trees_dict(dict): __slots__ = ('_running_eroot', '_target_eroot',) diff --git a/pym/portage/_global_updates.py b/pym/portage/_global_updates.py index 51750431d9c..c0f3df0b669 100644 --- a/pym/portage/_global_updates.py +++ b/pym/portage/_global_updates.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -32,11 +32,15 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): """ # only do this if we're root and not running repoman/ebuild digest - retupd = False if secpass < 2 or \ "SANDBOX_ACTIVE" in os.environ or \ len(trees) != 1: - return retupd + return False + + return _do_global_updates(trees, prev_mtimes, + quiet=quiet, if_mtime_changed=if_mtime_changed) + +def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): root = trees._running_eroot mysettings = trees[root]["vartree"].settings portdb = trees[root]["porttree"].dbapi @@ -61,6 +65,7 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): repo_map = {} timestamps = {} + retupd = False update_notice_printed = False for repo_name in portdb.getRepositories(): repo = portdb.getRepositoryPath(repo_name) @@ -237,8 +242,7 @@ def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): # Update progress above is indicated by characters written to stdout so # we print a couple new lines here to separate the progress output from # what follows. - print() - print() + writemsg_stdout("\n\n") if do_upgrade_packagesmessage and bindb and \ bindb.cpv_all(): diff --git a/pym/portage/_selinux.py b/pym/portage/_selinux.py index 9470978c4e4..17371451538 100644 --- a/pym/portage/_selinux.py +++ b/pym/portage/_selinux.py @@ -95,20 +95,32 @@ def setfscreate(ctx="\n"): raise OSError( _("setfscreate: Failed setting fs create context \"%s\".") % ctx) -def spawn_wrapper(spawn_func, selinux_type): - - selinux_type = _unicode_encode(selinux_type, - encoding=_encodings['content'], errors='strict') - - def wrapper_func(*args, **kwargs): - con = settype(selinux_type) - setexec(con) - try: - return spawn_func(*args, **kwargs) - finally: - setexec() - - return wrapper_func +class spawn_wrapper(object): + """ + Create a wrapper function for the given spawn function. When the wrapper + is called, it will adjust the arguments such that setexec() to be called + *after* the fork (thereby avoiding any interference with concurrent + threads in the calling process). + """ + __slots__ = ("_con", "_spawn_func") + + def __init__(self, spawn_func, selinux_type): + self._spawn_func = spawn_func + selinux_type = _unicode_encode(selinux_type, + encoding=_encodings['content'], errors='strict') + self._con = settype(selinux_type) + + def __call__(self, *args, **kwargs): + + pre_exec = kwargs.get("pre_exec") + + def _pre_exec(): + if pre_exec is not None: + pre_exec() + setexec(self._con) + + kwargs["pre_exec"] = _pre_exec + return self._spawn_func(*args, **kwargs) def symlink(target, link, reflnk): target = _unicode_encode(target, encoding=_encodings['fs'], errors='strict') diff --git a/pym/portage/_sets/__init__.py b/pym/portage/_sets/__init__.py index 88a4b3b1ba1..c3b590e928f 100644 --- a/pym/portage/_sets/__init__.py +++ b/pym/portage/_sets/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -111,16 +111,26 @@ class SetConfig(object): """ parser = self._parser + parser.remove_section("world") parser.add_section("world") parser.set("world", "class", "portage.sets.base.DummyPackageSet") parser.set("world", "packages", "@selected @system") + parser.remove_section("selected") parser.add_section("selected") parser.set("selected", "class", "portage.sets.files.WorldSelectedSet") + parser.remove_section("system") parser.add_section("system") parser.set("system", "class", "portage.sets.profiles.PackagesSystemSet") + parser.remove_section("usersets") + parser.add_section("usersets") + parser.set("usersets", "class", "portage.sets.files.StaticFileSet") + parser.set("usersets", "multiset", "true") + parser.set("usersets", "directory", "%(PORTAGE_CONFIGROOT)setc/portage/sets") + parser.set("usersets", "world-candidate", "true") + def update(self, setname, options): parser = self._parser self.errors = [] diff --git a/pym/portage/_sets/dbapi.py b/pym/portage/_sets/dbapi.py index 0f238f044e3..4982a92449f 100644 --- a/pym/portage/_sets/dbapi.py +++ b/pym/portage/_sets/dbapi.py @@ -1,10 +1,10 @@ -# Copyright 2007-2010 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import time from portage import os -from portage.versions import catpkgsplit, catsplit, pkgcmp, best +from portage.versions import best, catsplit, vercmp from portage.dep import Atom from portage.localization import _ from portage._sets.base import PackageSet @@ -72,18 +72,16 @@ class OwnerSet(PackageSet): aux_keys = ["SLOT"] if exclude_paths is None: for link, p in vardb._owners.iter_owners(paths): - cat, pn = catpkgsplit(link.mycpv)[:2] slot, = aux_get(link.mycpv, aux_keys) - rValue.add("%s/%s:%s" % (cat, pn, slot)) + rValue.add("%s:%s" % (link.mycpv.cp, slot)) else: all_paths = set() all_paths.update(paths) all_paths.update(exclude_paths) exclude_atoms = set() for link, p in vardb._owners.iter_owners(all_paths): - cat, pn = catpkgsplit(link.mycpv)[:2] slot, = aux_get(link.mycpv, aux_keys) - atom = "%s/%s:%s" % (cat, pn, slot) + atom = "%s:%s" % (link.mycpv.cp, slot) rValue.add(atom) if p in exclude_paths: exclude_atoms.add(atom) @@ -184,9 +182,7 @@ class DowngradeSet(PackageSet): ebuild = xmatch(xmatch_level, slot_atom) if not ebuild: continue - ebuild_split = catpkgsplit(ebuild)[1:] - installed_split = catpkgsplit(cpv)[1:] - if pkgcmp(installed_split, ebuild_split) > 0: + if vercmp(cpv.version, ebuild.version) > 0: atoms.append(slot_atom) self._setAtoms(atoms) diff --git a/pym/portage/_sets/files.py b/pym/portage/_sets/files.py index f19ecf6ce37..b891ea4f44a 100644 --- a/pym/portage/_sets/files.py +++ b/pym/portage/_sets/files.py @@ -1,4 +1,4 @@ -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -11,7 +11,6 @@ from portage import _unicode_decode from portage import _unicode_encode from portage.util import grabfile, write_atomic, ensure_dirs, normalize_path from portage.const import USER_CONFIG_PATH, WORLD_FILE, WORLD_SETS_FILE -from portage.const import _ENABLE_SET_CONFIG from portage.localization import _ from portage.locks import lockfile, unlockfile from portage import portage_gid @@ -231,9 +230,8 @@ class WorldSelectedSet(EditablePackageSet): write_atomic(self._filename, "".join(sorted("%s\n" % x for x in self._atoms))) - if _ENABLE_SET_CONFIG: - write_atomic(self._filename2, - "".join(sorted("%s\n" % x for x in self._nonatoms))) + write_atomic(self._filename2, + "".join(sorted("%s\n" % x for x in self._nonatoms))) def load(self): atoms = [] @@ -263,9 +261,8 @@ class WorldSelectedSet(EditablePackageSet): else: atoms.extend(self._atoms) - if _ENABLE_SET_CONFIG: - changed2, nonatoms = self._load2() - atoms_changed |= changed2 + changed2, nonatoms = self._load2() + atoms_changed |= changed2 if atoms_changed: self._setAtoms(atoms+nonatoms) diff --git a/pym/portage/_sets/security.py b/pym/portage/_sets/security.py index 2d8fcf667cc..7e856bc79d6 100644 --- a/pym/portage/_sets/security.py +++ b/pym/portage/_sets/security.py @@ -1,9 +1,9 @@ -# Copyright 2007 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage.glsa as glsa from portage._sets.base import PackageSet -from portage.versions import catpkgsplit, pkgcmp +from portage.versions import vercmp from portage._sets import get_boolean __all__ = ["SecuritySet", "NewGlsaSet", "NewAffectedSet", "AffectedSet"] @@ -45,12 +45,12 @@ class SecuritySet(PackageSet): for atom in atomlist[:]: cpv = self._portdbapi.xmatch("match-all", atom)[0] slot = self._portdbapi.aux_get(cpv, ["SLOT"])[0] - cps = "/".join(catpkgsplit(cpv)[0:2]) + ":" + slot + cps = "%s:%s" % (cpv.cp, slot) if not cps in mydict: mydict[cps] = (atom, cpv) else: other_cpv = mydict[cps][1] - if pkgcmp(catpkgsplit(cpv)[1:], catpkgsplit(other_cpv)[1:]) > 0: + if vercmp(cpv.version, other_cpv.version) > 0: atomlist.remove(mydict[cps][0]) mydict[cps] = (atom, cpv) return atomlist diff --git a/pym/portage/cache/fs_template.py b/pym/portage/cache/fs_template.py index a82e8623aca..8f0636ed013 100644 --- a/pym/portage/cache/fs_template.py +++ b/pym/portage/cache/fs_template.py @@ -1,14 +1,14 @@ -# Copyright: 2005 Gentoo Foundation +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 +import os as _os import sys from portage.cache import template from portage import os from portage.proxy.lazyimport import lazyimport lazyimport(globals(), - 'portage.data:portage_gid', 'portage.exception:PortageException', 'portage.util:apply_permissions', ) @@ -22,10 +22,6 @@ class FsBased(template.database): attempt to ensure files have the specified owners/perms""" def __init__(self, *args, **config): - """throws InitializationError if needs args aren't specified - gid and perms aren't listed do to an oddity python currying mechanism - gid=portage_gid - perms=0665""" for x, y in (("gid", -1), ("perms", -1)): if x in config: @@ -78,7 +74,17 @@ class FsBased(template.database): if self._perms != -1: os.umask(um) - + def _prune_empty_dirs(self): + all_dirs = [] + for parent, dirs, files in os.walk(self.location): + for x in dirs: + all_dirs.append(_os.path.join(parent, x)) + while all_dirs: + try: + _os.rmdir(all_dirs.pop()) + except OSError: + pass + def gen_label(base, label): """if supplied label is a path, generate a unique label based upon label, and supplied base path""" if label.find(os.path.sep) == -1: diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index fcc62ff94fd..a6a3e066d20 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -1,6 +1,7 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import re import sys from portage.cache import fs_template from portage.cache import cache_errors @@ -117,7 +118,13 @@ class database(fs_template.FsBased): for k, v in self._db_table.items(): if self._db_table_exists(v["table_name"]): create_statement = self._db_table_get_create(v["table_name"]) - if create_statement != v["create"]: + table_ok, missing_keys = self._db_validate_create_statement(create_statement) + if table_ok: + if missing_keys: + for k in sorted(missing_keys): + cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % + (self._db_table["packages"]["table_name"], k)) + else: writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) cursor.execute("DROP TABLE %s" % v["table_name"]) cursor.execute(v["create"]) @@ -138,6 +145,37 @@ class database(fs_template.FsBased): self._db_escape_string(table_name)) return cursor.fetchall()[0][0] + def _db_validate_create_statement(self, statement): + missing_keys = None + if statement == self._db_table["packages"]["create"]: + return True, missing_keys + + m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % + (self._db_table["packages"]["table_name"], + self._db_table["packages"]["package_id"]), + statement) + if m is None: + return False, missing_keys + + unique_constraints = set([self._db_table["packages"]["package_key"]]) + missing_keys = set(self._allowed_keys) + unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') + column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') + for x in m.group(1).split(","): + m = column_re.match(x) + if m is not None: + missing_keys.discard(m.group(1)) + continue + m = unique_re.match(x) + if m is not None: + unique_constraints.discard(m.group(1)) + continue + + if unique_constraints: + return False, missing_keys + + return True, missing_keys + def _db_init_cache_size(self, cache_bytes): cursor = self._db_cursor cursor.execute("PRAGMA page_size") diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py index 0af6c20cad3..cf1e8aebb0f 100644 --- a/pym/portage/cache/template.py +++ b/pym/portage/cache/template.py @@ -1,4 +1,4 @@ -# Copyright: 2005 Gentoo Foundation +# Copyright: 2005-2012 Gentoo Foundation # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 @@ -10,8 +10,11 @@ import warnings import operator if sys.hexversion >= 0x3000000: + _unicode = str basestring = str long = int +else: + _unicode = unicode class database(object): # this is for metadata/cache transfer. @@ -194,8 +197,13 @@ class database(object): def validate_entry(self, entry, ebuild_hash, eclass_db): hash_key = '_%s_' % self.validation_chf - if entry[hash_key] != getattr(ebuild_hash, self.validation_chf): + try: + entry_hash = entry[hash_key] + except KeyError: return False + else: + if entry_hash != getattr(ebuild_hash, self.validation_chf): + return False update = eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], self.validation_chf, self.store_eclass_paths) if update is None: @@ -268,7 +276,7 @@ def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True): # occasionally this occurs in the fs backends. they suck. return {} - converter = str + converter = _unicode if chf_type == 'mtime': converter = long diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py index 73e01b51980..daf4a0cbf03 100644 --- a/pym/portage/checksum.py +++ b/pym/portage/checksum.py @@ -137,8 +137,10 @@ try: except ImportError: pass +_whirlpool_unaccelerated = False if "WHIRLPOOL" not in hashfunc_map: # Bundled WHIRLPOOL implementation + _whirlpool_unaccelerated = True from portage.util.whirlpool import new as _new_whirlpool whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool, origin="bundled") @@ -186,7 +188,7 @@ def _perform_md5_merge(x, **kwargs): def perform_all(x, calc_prelink=0): mydict = {} for k in hashfunc_map: - mydict[k] = perform_checksum(x, hashfunc_map[k], calc_prelink)[0] + mydict[k] = perform_checksum(x, k, calc_prelink)[0] return mydict def get_valid_checksum_keys(): @@ -197,6 +199,24 @@ def get_hash_origin(hashtype): raise KeyError(hashtype) return hashorigin_map.get(hashtype, "unknown") +def _filter_unaccelarated_hashes(digests): + """ + If multiple digests are available and some are unaccelerated, + then return a new dict that omits the unaccelerated ones. This + allows extreme performance problems like bug #425046 to be + avoided whenever practical, especially for cases like stage + builds where acceleration may not be available for some hashes + due to minimization of dependencies. + """ + if _whirlpool_unaccelerated and "WHIRLPOOL" in digests: + verifiable_hash_types = set(digests).intersection(hashfunc_map) + verifiable_hash_types.discard("size") + if len(verifiable_hash_types) > 1: + digests = dict(digests) + digests.pop("WHIRLPOOL") + + return digests + def verify_all(filename, mydict, calc_prelink=0, strict=0): """ Verify all checksums against a file. diff --git a/pym/portage/const.py b/pym/portage/const.py index 5d6aabc0db4..ceef5c56b4f 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -1,5 +1,5 @@ # portage: Constants -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os @@ -90,19 +90,21 @@ SUPPORTED_FEATURES = frozenset([ "ccache", "chflags", "clean-logs", "collision-protect", "compress-build-logs", "compressdebug", "config-protect-if-modified", - "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", + "digest", "distcc", "distcc-pump", "distlocks", + "downgrade-backup", "ebuild-locks", "fakeroot", "fail-clean", "force-mirror", "force-prefix", "getbinpkg", "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", "metadata-transfer", "mirror", "multilib-strict", "news", "noauto", "noclean", "nodoc", "noinfo", "noman", "nostrip", "notitles", "parallel-fetch", "parallel-install", "parse-eapi-ebuild-head", - "prelink-checksums", "preserve-libs", + "prelink-checksums", "protect-owned", "python-trace", "sandbox", "selinux", "sesandbox", "sfperms", "sign", "skiprocheck", "split-elog", "split-log", "splitdebug", "strict", "stricter", "suidctl", "test", "test-fail-continue", "unknown-features-filter", "unknown-features-warn", + "unmerge-backup", "unmerge-logs", "unmerge-orphans", "userfetch", "userpriv", "usersandbox", "usersync", "webrsync-gpg", "xattr"]) @@ -112,13 +114,15 @@ HASHING_BLOCKSIZE = 32768 MANIFEST1_HASH_FUNCTIONS = ("MD5", "SHA256", "RMD160") MANIFEST1_REQUIRED_HASH = "MD5" -# Future events: +# Past events: # -# After WHIRLPOOL is supported in stable portage: -# - Add SHA256 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. -# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# 20120704 - After WHIRLPOOL is supported in stable portage: # - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows: # manifest-hashes = SHA256 SHA512 WHIRLPOOL +# - Add SHA512 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. +# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# +# Future events: # # After WHIRLPOOL is supported in stable portage for at least 1 year: # - Change MANIFEST2_REQUIRED_HASH to WHIRLPOOL. @@ -136,8 +140,8 @@ MANIFEST1_REQUIRED_HASH = "MD5" # After layout.conf settings correspond to defaults in stable portage: # - Remove redundant settings from gentoo-x86/metadata/layout.conf. -MANIFEST2_HASH_FUNCTIONS = ("RMD160", "SHA1", "SHA256", "SHA512", "WHIRLPOOL") -MANIFEST2_HASH_DEFAULTS = frozenset(["SHA1", "SHA256", "RMD160"]) +MANIFEST2_HASH_FUNCTIONS = ("SHA256", "SHA512", "WHIRLPOOL") +MANIFEST2_HASH_DEFAULTS = frozenset(["SHA256", "SHA512", "WHIRLPOOL"]) MANIFEST2_REQUIRED_HASH = "SHA256" MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") @@ -165,15 +169,13 @@ _ENABLE_DYN_LINK_MAP = True _ENABLE_PRESERVE_LIBS = True _ENABLE_REPO_NAME_WARN = True _ENABLE_SET_CONFIG = True +_ENABLE_INHERIT_CHECK = True # The definitions above will differ between branches, so it's useful to have # common lines of diff context here in order to avoid merge conflicts. -if not _ENABLE_PRESERVE_LIBS: +if _ENABLE_PRESERVE_LIBS: SUPPORTED_FEATURES = set(SUPPORTED_FEATURES) - SUPPORTED_FEATURES.remove("preserve-libs") + SUPPORTED_FEATURES.add("preserve-libs") SUPPORTED_FEATURES = frozenset(SUPPORTED_FEATURES) - -if not _ENABLE_SET_CONFIG: - WORLD_SETS_FILE = '/dev/null' diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index 96e85cecd92..b999fb5dfba 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ["dbapi"] @@ -8,14 +8,15 @@ import re import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.dbapi.dep_expand:dep_expand@_dep_expand', - 'portage.dep:match_from_list', + 'portage.dep:Atom,match_from_list,_match_slot', 'portage.output:colorize', 'portage.util:cmp_sort_key,writemsg', - 'portage.versions:catsplit,catpkgsplit,vercmp', + 'portage.versions:catsplit,catpkgsplit,vercmp,_pkg_str', ) from portage import os from portage import auxdbkeys +from portage.exception import InvalidData from portage.localization import _ class dbapi(object): @@ -24,6 +25,8 @@ class dbapi(object): _use_mutable = False _known_keys = frozenset(x for x in auxdbkeys if not x.startswith("UNUSED_0")) + _pkg_str_aux_keys = ("EAPI", "SLOT", "repository") + def __init__(self): pass @@ -46,7 +49,12 @@ class dbapi(object): def cp_list(self, cp, use_cache=1): raise NotImplementedError(self) - def _cpv_sort_ascending(self, cpv_list): + @staticmethod + def _cmp_cpv(cpv1, cpv2): + return vercmp(cpv1.version, cpv2.version) + + @staticmethod + def _cpv_sort_ascending(cpv_list): """ Use this to sort self.cp_list() results in ascending order. It sorts in place and returns None. @@ -55,12 +63,7 @@ class dbapi(object): # If the cpv includes explicit -r0, it has to be preserved # for consistency in findname and aux_get calls, so use a # dict to map strings back to their original values. - ver_map = {} - for cpv in cpv_list: - ver_map[cpv] = '-'.join(catpkgsplit(cpv)[2:]) - def cmp_cpv(cpv1, cpv2): - return vercmp(ver_map[cpv1], ver_map[cpv2]) - cpv_list.sort(key=cmp_sort_key(cmp_cpv)) + cpv_list.sort(key=cmp_sort_key(dbapi._cmp_cpv)) def cpv_all(self): """Return all CPVs in the db @@ -125,29 +128,53 @@ class dbapi(object): def _iter_match(self, atom, cpv_iter): cpv_iter = iter(match_from_list(atom, cpv_iter)) + if atom.repo: + cpv_iter = self._iter_match_repo(atom, cpv_iter) if atom.slot: cpv_iter = self._iter_match_slot(atom, cpv_iter) if atom.unevaluated_atom.use: cpv_iter = self._iter_match_use(atom, cpv_iter) - if atom.repo: - cpv_iter = self._iter_match_repo(atom, cpv_iter) return cpv_iter + def _pkg_str(self, cpv, repo): + """ + This is used to contruct _pkg_str instances on-demand during + matching. If cpv is a _pkg_str instance with slot attribute, + then simply return it. Otherwise, fetch metadata and construct + a _pkg_str instance. This may raise KeyError or InvalidData. + """ + try: + cpv.slot + except AttributeError: + pass + else: + return cpv + + metadata = dict(zip(self._pkg_str_aux_keys, + self.aux_get(cpv, self._pkg_str_aux_keys, myrepo=repo))) + + return _pkg_str(cpv, slot=metadata["SLOT"], + repo=metadata["repository"], eapi=metadata["EAPI"]) + def _iter_match_repo(self, atom, cpv_iter): for cpv in cpv_iter: try: - if self.aux_get(cpv, ["repository"], myrepo=atom.repo)[0] == atom.repo: - yield cpv - except KeyError: - continue + pkg_str = self._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + pass + else: + if pkg_str.repo == atom.repo: + yield pkg_str def _iter_match_slot(self, atom, cpv_iter): for cpv in cpv_iter: try: - if self.aux_get(cpv, ["SLOT"], myrepo=atom.repo)[0] == atom.slot: - yield cpv - except KeyError: - continue + pkg_str = self._pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + pass + else: + if _match_slot(atom, pkg_str): + yield pkg_str def _iter_match_use(self, atom, cpv_iter): """ @@ -155,7 +182,7 @@ class dbapi(object): 2) Check enabled/disabled flag states. """ - aux_keys = ["IUSE", "SLOT", "USE"] + aux_keys = ["IUSE", "SLOT", "USE", "repository"] for cpv in cpv_iter: try: metadata = dict(zip(aux_keys, @@ -190,32 +217,35 @@ class dbapi(object): missing_disabled = atom.use.missing_disabled.difference(iuse) if atom.use.enabled: - if atom.use.enabled.intersection(missing_disabled): + if any(x in atom.use.enabled for x in missing_disabled): return False need_enabled = atom.use.enabled.difference(use) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: + if any(x not in missing_enabled for x in need_enabled): return False if atom.use.disabled: - if atom.use.disabled.intersection(missing_enabled): + if any(x in atom.use.disabled for x in missing_enabled): return False need_disabled = atom.use.disabled.intersection(use) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: + if any(x not in missing_disabled for x in need_disabled): return False elif not self.settings.local_config: # Check masked and forced flags for repoman. - pkg = "%s:%s" % (cpv, metadata["SLOT"]) + if hasattr(cpv, 'slot'): + pkg = cpv + else: + pkg = _pkg_str(cpv, slot=metadata["SLOT"], + repo=metadata.get("repository")) usemask = self.settings._getUseMask(pkg) - if usemask.intersection(atom.use.enabled): + if any(x in usemask for x in atom.use.enabled): return False - useforce = self.settings._getUseForce(pkg).difference(usemask) - if useforce.intersection(atom.use.disabled): + useforce = self.settings._getUseForce(pkg) + if any(x in useforce and x not in usemask + for x in atom.use.disabled): return False return True @@ -245,17 +275,17 @@ class dbapi(object): maxval = len(cpv_all) aux_get = self.aux_get aux_update = self.aux_update - meta_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE", 'repository'] + meta_keys = ["DEPEND", "EAPI", "RDEPEND", "PDEPEND", "PROVIDE", 'repository'] repo_dict = None if isinstance(updates, dict): repo_dict = updates - from portage.update import update_dbentries if onUpdate: onUpdate(maxval, 0) if onProgress: onProgress(maxval, 0) for i, cpv in enumerate(cpv_all): metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) + eapi = metadata.pop('EAPI') repo = metadata.pop('repository') if repo_dict is None: updates_list = updates @@ -271,7 +301,8 @@ class dbapi(object): if not updates_list: continue - metadata_updates = update_dbentries(updates_list, metadata) + metadata_updates = \ + portage.update_dbentries(updates_list, metadata, eapi=eapi) if metadata_updates: aux_update(cpv, metadata_updates) if onUpdate: @@ -282,27 +313,39 @@ class dbapi(object): def move_slot_ent(self, mylist, repo_match=None): """This function takes a sequence: Args: - mylist: a sequence of (package, originalslot, newslot) + mylist: a sequence of (atom, originalslot, newslot) repo_match: callable that takes single repo_name argument and returns True if the update should be applied Returns: The number of slotmoves this function did """ - pkg = mylist[1] + atom = mylist[1] origslot = mylist[2] newslot = mylist[3] - origmatches = self.match(pkg) + + try: + atom.with_slot + except AttributeError: + atom = Atom(atom).with_slot(origslot) + else: + atom = atom.with_slot(origslot) + + origmatches = self.match(atom) moves = 0 if not origmatches: return moves for mycpv in origmatches: - slot = self.aux_get(mycpv, ["SLOT"])[0] - if slot != origslot: + try: + mycpv = self._pkg_str(mycpv, atom.repo) + except (KeyError, InvalidData): continue - if repo_match is not None \ - and not repo_match(self.aux_get(mycpv, ['repository'])[0]): + if repo_match is not None and not repo_match(mycpv.repo): continue moves += 1 + if "/" not in newslot and \ + mycpv.slot_abi and \ + mycpv.slot_abi not in (mycpv.slot, newslot): + newslot = "%s/%s" % (newslot, mycpv.slot_abi) mydata = {"SLOT": newslot+"\n"} self.aux_update(mycpv, mydata) return moves diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py index 2295b9f59f9..9527b0766d9 100644 --- a/pym/portage/dbapi/bintree.py +++ b/pym/portage/dbapi/bintree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ["bindbapi", "binarytree"] @@ -7,23 +7,23 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:dep_getkey,isjustname,match_from_list', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list', 'portage.output:EOutput,colorize', 'portage.locks:lockfile,unlockfile', - 'portage.package.ebuild.doebuild:_vdb_use_conditional_atoms', - 'portage.package.ebuild.fetch:_check_distfile', + 'portage.package.ebuild.fetch:_check_distfile,_hide_url_passwd', 'portage.update:update_dbentries', 'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \ 'writemsg,writemsg_stdout', 'portage.util.listdir:listdir', - 'portage.versions:best,catpkgsplit,catsplit', + 'portage.util._urlopen:urlopen@_urlopen', + 'portage.versions:best,catpkgsplit,catsplit,_pkg_str', ) from portage.cache.mappings import slot_dict_class from portage.const import CACHE_PATH from portage.dbapi.virtual import fakedbapi from portage.dep import Atom, use_reduce, paren_enclose -from portage.exception import AlarmSignal, InvalidPackageName, \ +from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \ PermissionDenied, PortageException from portage.localization import _ from portage import _movefile @@ -35,7 +35,6 @@ from portage import _unicode_encode import codecs import errno import io -import re import stat import subprocess import sys @@ -45,14 +44,15 @@ import warnings from itertools import chain try: from urllib.parse import urlparse - from urllib.request import urlopen as urllib_request_urlopen except ImportError: from urlparse import urlparse - from urllib import urlopen as urllib_request_urlopen if sys.hexversion >= 0x3000000: + _unicode = str basestring = str long = int +else: + _unicode = unicode class bindbapi(fakedbapi): _known_keys = frozenset(list(fakedbapi._known_keys) + \ @@ -221,6 +221,13 @@ def _pkgindex_cpv_map_latest_build(pkgindex): for d in pkgindex.packages: cpv = d["CPV"] + try: + cpv = _pkg_str(cpv) + except InvalidData: + writemsg(_("!!! Invalid remote binary package: %s\n") % cpv, + noiselevel=-1) + continue + btime = d.get('BUILD_TIME', '') try: btime = int(btime) @@ -237,7 +244,7 @@ def _pkgindex_cpv_map_latest_build(pkgindex): if other_btime and (not btime or other_btime > btime): continue - cpv_map[cpv] = d + cpv_map[_pkg_str(cpv)] = d return cpv_map @@ -373,15 +380,24 @@ class binarytree(object): if not origmatches: return moves for mycpv in origmatches: + try: + mycpv = self.dbapi._pkg_str(mycpv, None) + except (KeyError, InvalidData): + continue mycpv_cp = portage.cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue if repo_match is not None \ - and not repo_match(self.dbapi.aux_get(mycpv, - ['repository'])[0]): + and not repo_match(mycpv.repo): + continue + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if not isvalidatom(newcp, eapi=mycpv.eapi): continue - mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) + + mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) myoldpkg = catsplit(mycpv)[1] mynewpkg = catsplit(mynewcpv)[1] @@ -400,7 +416,7 @@ class binarytree(object): moves += 1 mytbz2 = portage.xpak.tbz2(tbz2path) mydata = mytbz2.get_data() - updated_items = update_dbentries([mylist], mydata) + updated_items = update_dbentries([mylist], mydata, eapi=mycpv.eapi) mydata.update(updated_items) mydata[b'PF'] = \ _unicode_encode(mynewpkg + "\n", @@ -659,6 +675,7 @@ class binarytree(object): if mycpv in pkg_paths: # discard duplicates (All/ is preferred) continue + mycpv = _pkg_str(mycpv) pkg_paths[mycpv] = mypath # update the path if the package has been moved oldpath = d.get("PATH") @@ -734,6 +751,7 @@ class binarytree(object): (mycpv, self.settings["PORTAGE_CONFIGROOT"]), noiselevel=-1) continue + mycpv = _pkg_str(mycpv) pkg_paths[mycpv] = mypath self.dbapi.cpv_inject(mycpv) update_pkgindex = True @@ -843,7 +861,7 @@ class binarytree(object): # slash, so join manually... url = base_url.rstrip("/") + "/Packages" try: - f = urllib_request_urlopen(url) + f = _urlopen(url) except IOError: path = parsed_url.path.rstrip("/") + "/Packages" @@ -915,7 +933,7 @@ class binarytree(object): noiselevel=-1) except EnvironmentError as e: writemsg(_("\n\n!!! Error fetching binhost package" \ - " info from '%s'\n") % base_url) + " info from '%s'\n") % _hide_url_passwd(base_url)) writemsg("!!! %s\n\n" % str(e)) del e pkgindex = None @@ -991,7 +1009,7 @@ class binarytree(object): writemsg_stdout("\n") writemsg_stdout( colorize("GOOD", _("Fetching bininfo from ")) + \ - re.sub(r'//(.+):.+@(.+)/', r'//\1:*password*@\2/', base_url) + "\n") + _hide_url_passwd(base_url) + "\n") remotepkgs = portage.getbinpkg.dir_get_metadata( base_url, chunk_size=chunk_size) @@ -1003,7 +1021,12 @@ class binarytree(object): noiselevel=-1) continue mycat = mycat.strip() - fullpkg = mycat+"/"+mypkg[:-5] + try: + fullpkg = _pkg_str(mycat+"/"+mypkg[:-5]) + except InvalidData: + writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg, + noiselevel=-1) + continue if fullpkg in metadata: # When using this old protocol, comparison with the remote @@ -1301,6 +1324,8 @@ class binarytree(object): """Returns the URI to the Packages file for a given package.""" return self._pkgindex_uri.get(pkgname) + + def gettbz2(self, pkgname): """Fetches the package from a remote site, if necessary. Attempts to resume if the file appears to be partially downloaded.""" @@ -1308,7 +1333,7 @@ class binarytree(object): tbz2name = os.path.basename(tbz2_path) resume = False if os.path.exists(tbz2_path): - if (tbz2name not in self.invalids): + if tbz2name[:-5] not in self.invalids: return else: resume = True diff --git a/pym/portage/dbapi/porttree.py b/pym/portage/dbapi/porttree.py index 382bcda4363..945c22c3dec 100644 --- a/pym/portage/dbapi/porttree.py +++ b/pym/portage/dbapi/porttree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -10,11 +10,11 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.checksum', 'portage.data:portage_gid,secpass', 'portage.dbapi.dep_expand:dep_expand', - 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce', + 'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot', 'portage.package.ebuild.doebuild:doebuild', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', - 'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp', + 'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str', ) from portage.cache import volatile @@ -22,7 +22,8 @@ from portage.cache.cache_errors import CacheError from portage.cache.mappings import Mapping from portage.dbapi import dbapi from portage.exception import PortageException, \ - FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName + FileNotFound, InvalidAtom, InvalidData, \ + InvalidDependString, InvalidPackageName from portage.localization import _ from portage import eclass_cache, \ @@ -36,8 +37,6 @@ from _emerge.EbuildMetadataPhase import EbuildMetadataPhase from _emerge.PollScheduler import PollScheduler import os as _os -import io -import stat import sys import traceback import warnings @@ -97,6 +96,7 @@ class portdbapi(dbapi): # this purpose because doebuild makes many changes to the config # instance that is passed in. self.doebuild_settings = config(clone=self.settings) + self._scheduler = PollScheduler().sched_iface self.depcachedir = os.path.realpath(self.settings.depcachedir) if os.environ.get("SANDBOX_ON") == "1": @@ -326,34 +326,7 @@ class portdbapi(dbapi): return (filename, x) return (None, 0) - def _metadata_process(self, cpv, ebuild_path, repo_path): - """ - Create an EbuildMetadataPhase instance to generate metadata for the - give ebuild. - @rtype: EbuildMetadataPhase - @return: A new EbuildMetadataPhase instance, or None if the - metadata cache is already valid. - """ - metadata, ebuild_hash = self._pull_valid_cache(cpv, ebuild_path, repo_path) - if metadata is not None: - return None - - process = EbuildMetadataPhase(cpv=cpv, - ebuild_hash=ebuild_hash, metadata_callback=self._metadata_callback, - portdb=self, repo_path=repo_path, settings=self.doebuild_settings) - return process - - def _metadata_callback(self, cpv, repo_path, metadata, ebuild_hash): - - i = metadata - if hasattr(metadata, "items"): - i = iter(metadata.items()) - metadata = dict(i) - - if metadata.get("INHERITED", False): - metadata["_eclasses_"] = self.repositories.get_repo_for_location(repo_path).eclass_db.get_eclass_data(metadata["INHERITED"].split()) - else: - metadata["_eclasses_"] = {} + def _write_cache(self, cpv, repo_path, metadata, ebuild_hash): try: cache = self.auxdb[repo_path] @@ -365,20 +338,6 @@ class portdbapi(dbapi): traceback.print_exc() cache = None - metadata.pop("INHERITED", None) - - eapi = metadata.get("EAPI") - if not eapi or not eapi.strip(): - eapi = "0" - metadata["EAPI"] = eapi - if not eapi_is_supported(eapi): - keys = set(metadata) - keys.discard('_eclasses_') - keys.discard('_mtime_') - keys.discard('_%s_' % chf) - metadata.update((k, '') for k in keys) - metadata["EAPI"] = "-" + eapi.lstrip("-") - if cache is not None: try: cache[cpv] = metadata @@ -386,7 +345,6 @@ class portdbapi(dbapi): # Normally this shouldn't happen, so we'll show # a traceback for debugging purposes. traceback.print_exc() - return metadata def _pull_valid_cache(self, cpv, ebuild_path, repo_path): try: @@ -429,7 +387,10 @@ class portdbapi(dbapi): if not eapi: eapi = '0' metadata['EAPI'] = eapi - if eapi[:1] == '-' and eapi_is_supported(eapi[1:]): + if not eapi_is_supported(eapi): + # Since we're supposed to be able to efficiently obtain the + # EAPI from _parse_eapi_ebuild_head, we disregard cache entries + # for unsupported EAPIs. continue if auxdb.validate_entry(metadata, ebuild_hash, eclass_db): break @@ -484,39 +445,19 @@ class portdbapi(dbapi): if myebuild in self._broken_ebuilds: raise KeyError(mycpv) - self.doebuild_settings.setcpv(mycpv) - eapi = None - - if eapi is None and \ - 'parse-eapi-ebuild-head' in self.doebuild_settings.features: - with io.open(_unicode_encode(myebuild, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - eapi = portage._parse_eapi_ebuild_head(f) + proc = EbuildMetadataPhase(cpv=mycpv, + ebuild_hash=ebuild_hash, portdb=self, + repo_path=mylocation, scheduler=self._scheduler, + settings=self.doebuild_settings) - if eapi is not None: - self.doebuild_settings.configdict['pkg']['EAPI'] = eapi + proc.start() + proc.wait() - if eapi is not None and not portage.eapi_is_supported(eapi): - mydata = self._metadata_callback( - mycpv, mylocation, {'EAPI':eapi}, ebuild_hash) - else: - proc = EbuildMetadataPhase(cpv=mycpv, eapi=eapi, - ebuild_hash=ebuild_hash, - metadata_callback=self._metadata_callback, portdb=self, - repo_path=mylocation, - scheduler=PollScheduler().sched_iface, - settings=self.doebuild_settings) - - proc.start() - proc.wait() - - if proc.returncode != os.EX_OK: - self._broken_ebuilds.add(myebuild) - raise KeyError(mycpv) + if proc.returncode != os.EX_OK: + self._broken_ebuilds.add(myebuild) + raise KeyError(mycpv) - mydata = proc.metadata + mydata = proc.metadata mydata["repository"] = self.repositories.get_name_for_location(mylocation) mydata["_mtime_"] = ebuild_hash.mtime @@ -570,7 +511,7 @@ class portdbapi(dbapi): # since callers already handle it. raise portage.exception.InvalidDependString( "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ - (mypkg, eapi.lstrip("-"))) + (mypkg, eapi)) return _parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris}, use=useflags) @@ -715,7 +656,8 @@ class portdbapi(dbapi): return l def cp_list(self, mycp, use_cache=1, mytree=None): - + # NOTE: Cache can be safely shared with the match cache, since the + # match cache uses the result from dep_expand for the cache_key. if self.frozen and mytree is not None \ and len(self.porttrees) == 1 \ and mytree == self.porttrees[0]: @@ -728,10 +670,8 @@ class portdbapi(dbapi): if cachelist is not None: # Try to propagate this to the match-all cache here for # repoman since he uses separate match-all caches for each - # profile (due to old-style virtuals). Do not propagate - # old-style virtuals since cp_list() doesn't expand them. - if not (not cachelist and mycp.startswith("virtual/")): - self.xcache["match-all"][mycp] = cachelist + # profile (due to differences in _get_implicit_iuse). + self.xcache["match-all"][(mycp, mycp)] = cachelist return cachelist[:] mysplit = mycp.split("/") invalid_category = mysplit[0] not in self._categories @@ -769,7 +709,7 @@ class portdbapi(dbapi): writemsg(_("\nInvalid ebuild version: %s\n") % \ os.path.join(oroot, mycp, x), noiselevel=-1) continue - d[mysplit[0]+"/"+pf] = None + d[_pkg_str(mysplit[0]+"/"+pf)] = None if invalid_category and d: writemsg(_("\n!!! '%s' has a category that is not listed in " \ "%setc/portage/categories\n") % \ @@ -783,10 +723,7 @@ class portdbapi(dbapi): if self.frozen and mytree is None: cachelist = mylist[:] self.xcache["cp-list"][mycp] = cachelist - # Do not propagate old-style virtuals since - # cp_list() doesn't expand them. - if not (not cachelist and mycp.startswith("virtual/")): - self.xcache["match-all"][mycp] = cachelist + self.xcache["match-all"][(mycp, mycp)] = cachelist return mylist def freeze(self): @@ -809,19 +746,21 @@ class portdbapi(dbapi): "has been renamed to match-visible", DeprecationWarning, stacklevel=2) - #if no updates are being made to the tree, we can consult our xcache... - if self.frozen: - try: - return self.xcache[level][origdep][:] - except KeyError: - pass - if mydep is None: #this stuff only runs on first call of xmatch() #create mydep, mykey from origdep mydep = dep_expand(origdep, mydb=self, settings=self.settings) mykey = mydep.cp + #if no updates are being made to the tree, we can consult our xcache... + cache_key = None + if self.frozen: + cache_key = (mydep, mydep.unevaluated_atom) + try: + return self.xcache[level][cache_key][:] + except KeyError: + pass + myval = None mytree = None if mydep.repo is not None: @@ -887,18 +826,24 @@ class portdbapi(dbapi): # ebuild not in this repo, or masked by corruption continue - if visibility_filter and not self._visible(cpv, metadata): + try: + pkg_str = _pkg_str(cpv, slot=metadata["SLOT"], + repo=metadata["repository"], eapi=metadata["EAPI"]) + except InvalidData: + continue + + if visibility_filter and not self._visible(pkg_str, metadata): continue if mydep.slot is not None and \ - mydep.slot != metadata["SLOT"]: + not _match_slot(mydep, pkg_str): continue if mydep.unevaluated_atom.use is not None and \ - not self._match_use(mydep, cpv, metadata): + not self._match_use(mydep, pkg_str, metadata): continue - myval.append(cpv) + myval.append(pkg_str) # only yield a given cpv once break @@ -930,10 +875,9 @@ class portdbapi(dbapi): if self.frozen: xcache_this_level = self.xcache.get(level) if xcache_this_level is not None: - xcache_this_level[mydep] = myval - if origdep and origdep != mydep: - xcache_this_level[origdep] = myval - myval = myval[:] + xcache_this_level[cache_key] = myval + if not isinstance(myval, _pkg_str): + myval = myval[:] return myval diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 25ea4c189cb..ea62f6bcc72 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -11,8 +11,9 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.data:portage_gid,portage_uid,secpass', 'portage.dbapi.dep_expand:dep_expand', 'portage.dbapi._MergeProcess:MergeProcess', - 'portage.dep:dep_getkey,isjustname,match_from_list,' + \ - 'use_reduce,_slot_re', + 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \ + 'use_reduce,_get_slot_re', + 'portage.eapi:_get_eapi_attrs', 'portage.elog:collect_ebuild_messages,collect_messages,' + \ 'elog_process,_merge_logentries', 'portage.locks:lockdir,unlockdir,lockfile,unlockfile', @@ -20,6 +21,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.doebuild:doebuild_environment,' + \ '_merge_unicode_error', '_spawn_phase', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', 'portage.update:fixdbentries', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ @@ -30,8 +32,8 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.movefile:movefile', 'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry', 'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap', - 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,pkgcmp,' + \ - '_pkgsplit@pkgsplit', + 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \ + '_pkgsplit@pkgsplit,_pkg_str', 'subprocess', 'tarfile', ) @@ -61,8 +63,10 @@ from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog from _emerge.PollScheduler import PollScheduler from _emerge.MiscFunctionsProcess import MiscFunctionsProcess +from _emerge.SpawnProcess import SpawnProcess import errno +import fnmatch import gc import grp import io @@ -86,6 +90,9 @@ except ImportError: if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class vardbapi(dbapi): @@ -314,14 +321,24 @@ class vardbapi(dbapi): if not origmatches: return moves for mycpv in origmatches: + try: + mycpv = self._pkg_str(mycpv, None) + except (KeyError, InvalidData): + continue mycpv_cp = cpv_getkey(mycpv) if mycpv_cp != origcp: # Ignore PROVIDE virtual match. continue if repo_match is not None \ - and not repo_match(self.aux_get(mycpv, ['repository'])[0]): + and not repo_match(mycpv.repo): + continue + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if not isvalidatom(newcp, eapi=mycpv.eapi): continue - mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) + + mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) mynewcat = catsplit(newcp)[0] origpath = self.getpath(mycpv) if not os.path.exists(origpath): @@ -351,7 +368,7 @@ class vardbapi(dbapi): del e write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") - fixdbentries([mylist], newpath) + fixdbentries([mylist], newpath, eapi=mycpv.eapi) return moves def cp_list(self, mycp, use_cache=1): @@ -385,7 +402,7 @@ class vardbapi(dbapi): continue if len(mysplit) > 1: if ps[0] == mysplit[1]: - returnme.append(mysplit[0]+"/"+x) + returnme.append(_pkg_str(mysplit[0]+"/"+x)) self._cpv_sort_ascending(returnme) if use_cache: self.cpcache[mycp] = [mystat, returnme[:]] @@ -484,6 +501,7 @@ class vardbapi(dbapi): "caching match function" mydep = dep_expand( origdep, mydb=self, use_cache=use_cache, settings=self.settings) + cache_key = (mydep, mydep.unevaluated_atom) mykey = dep_getkey(mydep) mycat = catsplit(mykey)[0] if not use_cache: @@ -505,8 +523,8 @@ class vardbapi(dbapi): if mydep not in self.matchcache[mycat]: mymatch = list(self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))) - self.matchcache[mycat][mydep] = mymatch - return self.matchcache[mycat][mydep][:] + self.matchcache[mycat][cache_key] = mymatch + return self.matchcache[mycat][cache_key][:] def findname(self, mycpv, myrepo=None): return self.getpath(str(mycpv), filename=catsplit(mycpv)[1]+".ebuild") @@ -639,7 +657,8 @@ class vardbapi(dbapi): if e.errno != errno.ENOENT: raise raise KeyError(mycpv) - mydir_mtime = mydir_stat[stat.ST_MTIME] + # Use float mtime when available. + mydir_mtime = mydir_stat.st_mtime pkg_data = self._aux_cache["packages"].get(mycpv) pull_me = cache_these.union(wants) mydata = {"_mtime_" : mydir_mtime} @@ -652,13 +671,18 @@ class vardbapi(dbapi): pkg_data = None else: cache_mtime, metadata = pkg_data - if not isinstance(cache_mtime, (long, int)) or \ + if not isinstance(cache_mtime, (float, long, int)) or \ not isinstance(metadata, dict): pkg_data = None if pkg_data: cache_mtime, metadata = pkg_data - cache_valid = cache_mtime == mydir_mtime + if isinstance(cache_mtime, float): + cache_valid = cache_mtime == mydir_stat.st_mtime + else: + # Cache may contain integer mtime. + cache_valid = cache_mtime == mydir_stat[stat.ST_MTIME] + if cache_valid: # Migrate old metadata to unicode. for k, v in metadata.items(): @@ -678,10 +702,12 @@ class vardbapi(dbapi): cache_data.update(metadata) for aux_key in cache_these: cache_data[aux_key] = mydata[aux_key] - self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) + self._aux_cache["packages"][_unicode(mycpv)] = \ + (mydir_mtime, cache_data) self._aux_cache["modified"].add(mycpv) - if _slot_re.match(mydata['SLOT']) is None: + eapi_attrs = _get_eapi_attrs(mydata['EAPI']) + if _get_slot_re(eapi_attrs).match(mydata['SLOT']) is None: # Empty or invalid slot triggers InvalidAtom exceptions when # generating slot atoms for packages, so translate it to '0' here. mydata['SLOT'] = _unicode_decode('0') @@ -1057,7 +1083,7 @@ class vardbapi(dbapi): counter = int(counter) except ValueError: counter = 0 - return (cpv, counter, mtime) + return (_unicode(cpv), counter, mtime) class _owners_db(object): @@ -1430,8 +1456,13 @@ class dblink(object): self.cat = cat self.pkg = pkg self.mycpv = self.cat + "/" + self.pkg - self.mysplit = list(catpkgsplit(self.mycpv)[1:]) - self.mysplit[0] = "%s/%s" % (self.cat, self.mysplit[0]) + if self.mycpv == settings.mycpv and \ + isinstance(settings.mycpv, _pkg_str): + self.mycpv = settings.mycpv + else: + self.mycpv = _pkg_str(self.mycpv) + self.mysplit = list(self.mycpv.cpv_split[1:]) + self.mysplit[0] = self.mycpv.cp self.treetype = treetype if vartree is None: vartree = portage.db[self._eroot]["vartree"] @@ -1452,7 +1483,7 @@ class dblink(object): self._contents_inodes = None self._contents_basenames = None self._linkmap_broken = False - self._md5_merge_map = {} + self._hardlink_merge_map = {} self._hash_key = (self._eroot, self.mycpv) self._protect_obj = None self._pipe = pipe @@ -1764,6 +1795,11 @@ class dblink(object): showMessage = self._display_merge if self.vartree.dbapi._categories is not None: self.vartree.dbapi._categories = None + + # When others_in_slot is not None, the backup has already been + # handled by the caller. + caller_handles_backup = others_in_slot is not None + # When others_in_slot is supplied, the security check has already been # done for this slot, so it shouldn't be repeated until the next # replacement or unmerge operation. @@ -1812,9 +1848,6 @@ class dblink(object): except UnsupportedAPIException as e: eapi_unsupported = e - self._prune_plib_registry(unmerge=True, needed=needed, - preserve_paths=preserve_paths) - builddir_lock = None scheduler = self._scheduler retval = os.EX_OK @@ -1829,6 +1862,19 @@ class dblink(object): prepare_build_dirs(settings=self.settings, cleanup=True) log_path = self.settings.get("PORTAGE_LOG_FILE") + # Do this before the following _prune_plib_registry call, since + # that removes preserved libraries from our CONTENTS, and we + # may want to backup those libraries first. + if not caller_handles_backup: + retval = self._pre_unmerge_backup(background) + if retval != os.EX_OK: + showMessage(_("!!! FAILED prerm: quickpkg: %s\n") % retval, + level=logging.ERROR, noiselevel=-1) + return retval + + self._prune_plib_registry(unmerge=True, needed=needed, + preserve_paths=preserve_paths) + # Log the error after PORTAGE_LOG_FILE is initialized # by prepare_build_dirs above. if eapi_unsupported: @@ -2051,7 +2097,9 @@ class dblink(object): #process symlinks second-to-last, directories last. mydirs = set() - modprotect = os.path.join(self._eroot, "lib/modules/") + + uninstall_ignore = portage.util.shlex_split( + self.settings.get("UNINSTALL_IGNORE", "")) def unlink(file_name, lstatobj): if bsd_chflags: @@ -2158,6 +2206,24 @@ class dblink(object): if lstatobj is None: show_unmerge("---", unmerge_desc["!found"], file_type, obj) continue + + f_match = obj[len(eroot)-1:] + ignore = False + for pattern in uninstall_ignore: + if fnmatch.fnmatch(f_match, pattern): + ignore = True + break + + if not ignore: + if islink and f_match in \ + ("/lib", "/usr/lib", "/usr/local/lib"): + # Ignore libdir symlinks for bug #423127. + ignore = True + + if ignore: + show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) + continue + # don't use EROOT, CONTENTS entries already contain EPREFIX if obj.startswith(real_root): relative_path = obj[real_root_len:] @@ -2167,8 +2233,9 @@ class dblink(object): is_owned = True break - if file_type == "sym" and is_owned and \ - (islink and statobj and stat.S_ISDIR(statobj.st_mode)): + if is_owned and islink and \ + file_type in ("sym", "dir") and \ + statobj and stat.S_ISDIR(statobj.st_mode): # A new instance of this package claims the file, so # don't unmerge it. If the file is symlink to a # directory and the unmerging package installed it as @@ -2200,18 +2267,6 @@ class dblink(object): continue elif relative_path in cfgfiledict: stale_confmem.append(relative_path) - # next line includes a tweak to protect modules from being unmerged, - # but we don't protect modules from being overwritten if they are - # upgraded. We effectively only want one half of the config protection - # functionality for /lib/modules. For portage-ng both capabilities - # should be able to be independently specified. - # TODO: For rebuilds, re-parent previous modules to the new - # installed instance (so they are not orphans). For normal - # uninstall (not rebuild/reinstall), remove the modules along - # with all other files (leave no orphans). - if obj.startswith(modprotect): - show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) - continue # Don't unlink symlinks to directories here since that can # remove /lib and /usr/lib symlinks. @@ -2233,12 +2288,12 @@ class dblink(object): show_unmerge("---", unmerge_desc["!mtime"], file_type, obj) continue - if pkgfiles[objkey][0] == "dir": + if file_type == "dir" and not islink: if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode): show_unmerge("---", unmerge_desc["!dir"], file_type, obj) continue mydirs.add((obj, (lstatobj.st_dev, lstatobj.st_ino))) - elif pkgfiles[objkey][0] == "sym": + elif file_type == "sym" or (file_type == "dir" and islink): if not islink: show_unmerge("---", unmerge_desc["!sym"], file_type, obj) continue @@ -2348,7 +2403,11 @@ class dblink(object): if protected_symlinks: msg = "One or more symlinks to directories have been " + \ "preserved in order to ensure that files installed " + \ - "via these symlinks remain accessible:" + "via these symlinks remain accessible. " + \ + "This indicates that the mentioned symlink(s) may " + \ + "be obsolete remnants of an old install, and it " + \ + "may be appropriate to replace a given symlink " + \ + "with the directory that it points to." lines = textwrap.wrap(msg, 72) lines.append("") flat_list = set() @@ -2358,7 +2417,7 @@ class dblink(object): lines.append("\t%s" % (os.path.join(real_root, f.lstrip(os.sep)))) lines.append("") - self._elog("eerror", "postrm", lines) + self._elog("elog", "postrm", lines) # Remove stale entries from config memory. if stale_confmem: @@ -3075,9 +3134,13 @@ class dblink(object): os = _os_merge - collision_ignore = set([normalize_path(myignore) for myignore in \ - portage.util.shlex_split( - self.settings.get("COLLISION_IGNORE", ""))]) + collision_ignore = [] + for x in portage.util.shlex_split( + self.settings.get("COLLISION_IGNORE", "")): + if os.path.isdir(os.path.join(self._eroot, x.lstrip(os.sep))): + x = normalize_path(x) + x += "/*" + collision_ignore.append(x) # For collisions with preserved libraries, the current package # will assume ownership and the libraries will be unregistered. @@ -3178,15 +3241,12 @@ class dblink(object): if not isowned and self.isprotected(full_path): isowned = True if not isowned: + f_match = full_path[len(self._eroot)-1:] stopmerge = True - if collision_ignore: - if f in collision_ignore: + for pattern in collision_ignore: + if fnmatch.fnmatch(f_match, pattern): stopmerge = False - else: - for myignore in collision_ignore: - if f.startswith(myignore + os.path.sep): - stopmerge = False - break + break if stopmerge: collisions.append(f) return collisions, symlink_collisions, plib_collisions @@ -3450,6 +3510,10 @@ class dblink(object): if not os.path.exists(self.dbcatdir): ensure_dirs(self.dbcatdir) + # NOTE: We use SLOT obtained from the inforoot + # directory, in order to support USE=multislot. + # Use _pkg_str discard the sub-slot part if necessary. + slot = _pkg_str(self.mycpv, slot=slot).slot cp = self.mysplit[0] slot_atom = "%s:%s" % (cp, slot) @@ -3802,6 +3866,20 @@ class dblink(object): self.delete() ensure_dirs(self.dbtmpdir) + downgrade = False + if self._installed_instance is not None and \ + vercmp(self.mycpv.version, + self._installed_instance.mycpv.version) < 0: + downgrade = True + + if self._installed_instance is not None: + rval = self._pre_merge_backup(self._installed_instance, downgrade) + if rval != os.EX_OK: + showMessage(_("!!! FAILED preinst: ") + + "quickpkg: %s\n" % rval, + level=logging.ERROR, noiselevel=-1) + return rval + # run preinst script showMessage(_(">>> Merging %(cpv)s to %(destroot)s\n") % \ {"cpv":self.mycpv, "destroot":destroot}) @@ -3835,21 +3913,15 @@ class dblink(object): #if we have a file containing previously-merged config file md5sums, grab it. self.vartree.dbapi._fs_lock() try: + # Always behave like --noconfmem is enabled for downgrades + # so that people who don't know about this option are less + # likely to get confused when doing upgrade/downgrade cycles. cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file) - if "NOCONFMEM" in self.settings: + if "NOCONFMEM" in self.settings or downgrade: cfgfiledict["IGNORE"]=1 else: cfgfiledict["IGNORE"]=0 - # Always behave like --noconfmem is enabled for downgrades - # so that people who don't know about this option are less - # likely to get confused when doing upgrade/downgrade cycles. - pv_split = catpkgsplit(self.mycpv)[1:] - for other in others_in_slot: - if pkgcmp(pv_split, catpkgsplit(other.mycpv)[1:]) < 0: - cfgfiledict["IGNORE"] = 1 - break - rval = self._merge_contents(srcroot, destroot, cfgfiledict) if rval != os.EX_OK: return rval @@ -4276,7 +4348,7 @@ class dblink(object): myabsto = myabsto.lstrip(sep) if self.settings and self.settings["D"]: if myto.startswith(self.settings["D"]): - myto = myto[len(self.settings["D"]):] + myto = myto[len(self.settings["D"])-1:] # myrealto contains the path of the real file to which this symlink points. # we can simply test for existence of this file to see if the target has been merged yet myrealto = normalize_path(os.path.join(destroot, myabsto)) @@ -4500,10 +4572,10 @@ class dblink(object): # as hardlinks (having identical st_dev and st_ino). hardlink_key = (mystat.st_dev, mystat.st_ino) - hardlink_candidates = self._md5_merge_map.get(hardlink_key) + hardlink_candidates = self._hardlink_merge_map.get(hardlink_key) if hardlink_candidates is None: hardlink_candidates = [] - self._md5_merge_map[hardlink_key] = hardlink_candidates + self._hardlink_merge_map[hardlink_key] = hardlink_candidates mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, @@ -4511,8 +4583,7 @@ class dblink(object): encoding=_encodings['merge']) if mymtime is None: return 1 - if hardlink_candidates is not None: - hardlink_candidates.append(mydest) + hardlink_candidates.append(mydest) zing = ">>>" if mymtime != None: @@ -4653,6 +4724,66 @@ class dblink(object): "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" return os.path.exists(os.path.join(self.dbdir, "CATEGORY")) + def _pre_merge_backup(self, backup_dblink, downgrade): + + if ("unmerge-backup" in self.settings.features or + (downgrade and "downgrade-backup" in self.settings.features)): + return self._quickpkg_dblink(backup_dblink, False, None) + + return os.EX_OK + + def _pre_unmerge_backup(self, background): + + if "unmerge-backup" in self.settings.features : + logfile = None + if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": + logfile = self.settings.get("PORTAGE_LOG_FILE") + return self._quickpkg_dblink(self, background, logfile) + + return os.EX_OK + + def _quickpkg_dblink(self, backup_dblink, background, logfile): + + trees = QueryCommand.get_db()[self.settings["EROOT"]] + bintree = trees["bintree"] + binpkg_path = bintree.getname(backup_dblink.mycpv) + if os.path.exists(binpkg_path) and \ + catsplit(backup_dblink.mycpv)[1] not in bintree.invalids: + return os.EX_OK + + self.lockdb() + try: + + if not backup_dblink.exists(): + # It got unmerged by a concurrent process. + return os.EX_OK + + # Call quickpkg for support of QUICKPKG_DEFAULT_OPTS and stuff. + quickpkg_binary = os.path.join(self.settings["PORTAGE_BIN_PATH"], + "quickpkg") + + # Let quickpkg inherit the global vartree config's env. + env = dict(self.vartree.settings.items()) + env["__PORTAGE_INHERIT_VARDB_LOCK"] = "1" + + pythonpath = [x for x in env.get('PYTHONPATH', '').split(":") if x] + if not pythonpath or \ + not os.path.samefile(pythonpath[0], portage._pym_path): + pythonpath.insert(0, portage._pym_path) + env['PYTHONPATH'] = ":".join(pythonpath) + + quickpkg_proc = SpawnProcess( + args=[portage._python_interpreter, quickpkg_binary, + "=%s" % (backup_dblink.mycpv,)], + background=background, env=env, + scheduler=self._scheduler, logfile=logfile) + quickpkg_proc.start() + + return quickpkg_proc.wait() + + finally: + self.unlockdb() + def merge(mycat, mypkg, pkgloc, infloc, myroot=None, settings=None, myebuild=None, mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None, diff --git a/pym/portage/dbapi/virtual.py b/pym/portage/dbapi/virtual.py index ec97ffed609..213708c93e5 100644 --- a/pym/portage/dbapi/virtual.py +++ b/pym/portage/dbapi/virtual.py @@ -1,9 +1,10 @@ -# Copyright 1998-2007 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.dbapi import dbapi -from portage import cpv_getkey +from portage.dbapi.dep_expand import dep_expand +from portage.versions import cpv_getkey, _pkg_str class fakedbapi(dbapi): """A fake dbapi that allows consumers to inject/remove packages to/from it @@ -31,27 +32,30 @@ class fakedbapi(dbapi): self._match_cache = {} def match(self, origdep, use_cache=1): - result = self._match_cache.get(origdep, None) + atom = dep_expand(origdep, mydb=self, settings=self.settings) + cache_key = (atom, atom.unevaluated_atom) + result = self._match_cache.get(cache_key) if result is not None: return result[:] - result = dbapi.match(self, origdep, use_cache=use_cache) - self._match_cache[origdep] = result + result = list(self._iter_match(atom, self.cp_list(atom.cp))) + self._match_cache[cache_key] = result return result[:] def cpv_exists(self, mycpv, myrepo=None): return mycpv in self.cpvdict def cp_list(self, mycp, use_cache=1, myrepo=None): - cachelist = self._match_cache.get(mycp) - # cp_list() doesn't expand old-style virtuals - if cachelist and cachelist[0].startswith(mycp): + # NOTE: Cache can be safely shared with the match cache, since the + # match cache uses the result from dep_expand for the cache_key. + cache_key = (mycp, mycp) + cachelist = self._match_cache.get(cache_key) + if cachelist is not None: return cachelist[:] cpv_list = self.cpdict.get(mycp) if cpv_list is None: cpv_list = [] self._cpv_sort_ascending(cpv_list) - if not (not cpv_list and mycp.startswith("virtual/")): - self._match_cache[mycp] = cpv_list + self._match_cache[cache_key] = cpv_list return cpv_list[:] def cp_all(self): @@ -70,24 +74,55 @@ class fakedbapi(dbapi): @param metadata: dict """ self._clear_cache() - mycp = cpv_getkey(mycpv) + + try: + mycp = mycpv.cp + except AttributeError: + mycp = None + try: + myslot = mycpv.slot + except AttributeError: + myslot = None + + if mycp is None or \ + (myslot is None and metadata is not None and metadata.get('SLOT')): + if metadata is None: + mycpv = _pkg_str(mycpv) + else: + mycpv = _pkg_str(mycpv, slot=metadata.get('SLOT'), + repo=metadata.get('repository'), eapi=metadata.get('EAPI')) + + mycp = mycpv.cp + try: + myslot = mycpv.slot + except AttributeError: + pass + self.cpvdict[mycpv] = metadata - myslot = None - if self._exclusive_slots and metadata: - myslot = metadata.get("SLOT", None) + if not self._exclusive_slots: + myslot = None if myslot and mycp in self.cpdict: # If necessary, remove another package in the same SLOT. for cpv in self.cpdict[mycp]: if mycpv != cpv: - other_metadata = self.cpvdict[cpv] - if other_metadata: - if myslot == other_metadata.get("SLOT", None): + try: + other_slot = cpv.slot + except AttributeError: + pass + else: + if myslot == other_slot: self.cpv_remove(cpv) break - if mycp not in self.cpdict: - self.cpdict[mycp] = [] - if not mycpv in self.cpdict[mycp]: - self.cpdict[mycp].append(mycpv) + + cp_list = self.cpdict.get(mycp) + if cp_list is None: + cp_list = [] + self.cpdict[mycp] = cp_list + try: + cp_list.remove(mycpv) + except ValueError: + pass + cp_list.append(mycpv) def cpv_remove(self,mycpv): """Removes a cpv from the list of available packages.""" diff --git a/pym/portage/dep/__init__.py b/pym/portage/dep/__init__.py index 3832b0bdeec..e547debd41c 100644 --- a/pym/portage/dep/__init__.py +++ b/pym/portage/dep/__init__.py @@ -13,47 +13,197 @@ __all__ = [ '_repo_separator', '_slot_separator', ] -# DEPEND SYNTAX: -# -# 'use?' only affects the immediately following word! -# Nesting is the only legal way to form multiple '[!]use?' requirements. -# -# Where: 'a' and 'b' are use flags, and 'z' is a depend atom. -# -# "a? z" -- If 'a' in [use], then b is valid. -# "a? ( z )" -- Syntax with parenthesis. -# "a? b? z" -- Deprecated. -# "a? ( b? z )" -- Valid -# "a? ( b? ( z ) ) -- Valid -# - import re, sys import warnings from itertools import chain import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util:cmp_sort_key', + 'portage.util:cmp_sort_key,writemsg', ) -from portage import _unicode_decode -from portage.eapi import eapi_has_slot_deps, eapi_has_src_uri_arrows, \ - eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_use_dep_defaults, \ - eapi_has_repo_deps +from portage import _encodings, _unicode_decode, _unicode_encode +from portage.eapi import _get_eapi_attrs from portage.exception import InvalidAtom, InvalidData, InvalidDependString from portage.localization import _ from portage.versions import catpkgsplit, catsplit, \ - pkgcmp, vercmp, ververify, _cp, _cpv + vercmp, ververify, _cp, _cpv, _pkg_str, _unknown_repo import portage.cache.mappings if sys.hexversion >= 0x3000000: basestring = str + _unicode = str +else: + _unicode = unicode # Api consumers included in portage should set this to True. # Once the relevant api changes are in a portage release with # stable keywords, make these warnings unconditional. _internal_warnings = False +# \w is [a-zA-Z0-9_] + +# PMS 3.1.3: A slot name may contain any of the characters [A-Za-z0-9+_.-]. +# It must not begin with a hyphen or a dot. +_slot_separator = ":" +_slot = r'([\w+][\w+.-]*)' +# loosly match SLOT, which may have an optional ABI part +_slot_loose = r'([\w+./*=-]+)' + +_use = r'\[.*\]' +_op = r'([=~]|[><]=?)' + +_repo_separator = "::" +_repo_name = r'[\w][\w-]*' +_repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' + +_extended_cat = r'[\w+*][\w+.*-]*' + +_slot_re_cache = {} + +def _get_slot_re(eapi_attrs): + cache_key = eapi_attrs.slot_abi + slot_re = _slot_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_abi: + slot_re = _slot + r'(/' + _slot + r'=?)?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE) + + _slot_re_cache[cache_key] = slot_re + return slot_re + +_slot_dep_re_cache = {} + +def _get_slot_dep_re(eapi_attrs): + cache_key = eapi_attrs.slot_abi + slot_re = _slot_dep_re_cache.get(cache_key) + if slot_re is not None: + return slot_re + + if eapi_attrs.slot_abi: + slot_re = _slot + r'?(\*|=|/' + _slot + r'=?)?' + else: + slot_re = _slot + + slot_re = re.compile('^' + slot_re + '$', re.VERBOSE) + + _slot_dep_re_cache[cache_key] = slot_re + return slot_re + +def _match_slot(atom, pkg): + if pkg.slot == atom.slot: + if not atom.slot_abi: + return True + elif atom.slot_abi == pkg.slot_abi: + return True + return False + +_atom_re_cache = {} + +def _get_atom_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + atom_re = _atom_re_cache.get(cache_key) + if atom_re is not None: + return atom_re + + if eapi_attrs.dots_in_PN: + cp_re = _cp['dots_allowed_in_PN'] + cpv_re = _cpv['dots_allowed_in_PN'] + else: + cp_re = _cp['dots_disallowed_in_PN'] + cpv_re = _cpv['dots_disallowed_in_PN'] + + atom_re = re.compile('^(?P<without_use>(?:' + + '(?P<op>' + _op + cpv_re + ')|' + + '(?P<star>=' + cpv_re + r'\*)|' + + '(?P<simple>' + cp_re + '))' + + '(' + _slot_separator + _slot_loose + ')?' + + _repo + ')(' + _use + ')?$', re.VERBOSE) + + _atom_re_cache[cache_key] = atom_re + return atom_re + +_atom_wildcard_re_cache = {} + +def _get_atom_wildcard_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + atom_re = _atom_wildcard_re_cache.get(cache_key) + if atom_re is not None: + return atom_re + + if eapi_attrs.dots_in_PN: + pkg_re = r'[\w+*][\w+.*-]*?' + else: + pkg_re = r'[\w+*][\w+*-]*?' + + atom_re = re.compile(r'((?P<simple>(' + + _extended_cat + r')/(' + pkg_re + r'))' + \ + '|(?P<star>=((' + _extended_cat + r')/(' + pkg_re + r'))-(?P<version>\*\d+\*)))' + \ + '(:(?P<slot>' + _slot_loose + r'))?(' + + _repo_separator + r'(?P<repo>' + _repo_name + r'))?$') + + _atom_wildcard_re_cache[cache_key] = atom_re + return atom_re + +_usedep_re_cache = {} + +def _get_usedep_re(eapi_attrs): + """ + @param eapi_attrs: The EAPI attributes from _get_eapi_attrs + @type eapi_attrs: _eapi_attrs + @rtype: regular expression object + @return: A regular expression object that matches valid USE deps for the + given eapi. + """ + cache_key = eapi_attrs.dots_in_use_flags + usedep_re = _usedep_re_cache.get(cache_key) + if usedep_re is not None: + return usedep_re + + if eapi_attrs.dots_in_use_flags: + _flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@.-]*' + else: + _flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@-]*' + + usedep_re = re.compile(r'^(?P<prefix>[!-]?)(?P<flag>' + + _flag_re + r')(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$') + + _usedep_re_cache[cache_key] = usedep_re + return usedep_re + +_useflag_re_cache = {} + +def _get_useflag_re(eapi): + """ + When eapi is None then validation is not as strict, since we want the + same to work for multiple EAPIs that may have slightly different rules. + @param eapi: The EAPI + @type eapi: String or None + @rtype: regular expression object + @return: A regular expression object that matches valid USE flags for the + given eapi. + """ + eapi_attrs = _get_eapi_attrs(eapi) + cache_key = eapi_attrs.dots_in_use_flags + useflag_re = _useflag_re_cache.get(cache_key) + if useflag_re is not None: + return useflag_re + + if eapi_attrs.dots_in_use_flags: + flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@.-]*' + else: + flag_re = r'[A-Za-z0-9][A-Za-z0-9+_@-]*' + + useflag_re = re.compile(r'^' + flag_re + r'$') + + _useflag_re_cache[cache_key] = useflag_re + return useflag_re + def cpvequal(cpv1, cpv2): """ @@ -74,16 +224,27 @@ def cpvequal(cpv1, cpv2): """ - split1 = catpkgsplit(cpv1) - split2 = catpkgsplit(cpv2) - - if not split1 or not split2: + try: + try: + split1 = cpv1.cpv_split + except AttributeError: + cpv1 = _pkg_str(cpv1) + split1 = cpv1.cpv_split + + try: + split2 = cpv2.cpv_split + except AttributeError: + cpv2 = _pkg_str(cpv2) + split2 = cpv2.cpv_split + + except InvalidData: raise portage.exception.PortageException(_("Invalid data '%s, %s', parameter was not a CPV") % (cpv1, cpv2)) - - if split1[0] != split2[0]: + + if split1[0] != split2[0] or \ + split1[1] != split2[1]: return False - - return (pkgcmp(split1[1:], split2[1:]) == 0) + + return vercmp(cpv1.version, cpv2.version) == 0 def strip_empty(myarr): """ @@ -239,7 +400,7 @@ class paren_normalize(list): self._zap_parens(x, dest) return dest -def paren_enclose(mylist, unevaluated_atom=False): +def paren_enclose(mylist, unevaluated_atom=False, opconvert=False): """ Convert a list to a string with sublists enclosed with parens. @@ -256,7 +417,10 @@ def paren_enclose(mylist, unevaluated_atom=False): mystrparts = [] for x in mylist: if isinstance(x, list): - mystrparts.append("( "+paren_enclose(x)+" )") + if opconvert and x and x[0] == "||": + mystrparts.append("%s ( %s )" % (x[0], paren_enclose(x[1:]))) + else: + mystrparts.append("( %s )" % paren_enclose(x)) else: if unevaluated_atom: x = getattr(x, 'unevaluated_atom', x) @@ -309,6 +473,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i if matchall and matchnone: raise ValueError("portage.dep.use_reduce: 'matchall' and 'matchnone' are mutually exclusive") + eapi_attrs = _get_eapi_attrs(eapi) useflag_re = _get_useflag_re(eapi) def is_active(conditional): @@ -524,7 +689,7 @@ def use_reduce(depstr, uselist=[], masklist=[], matchall=False, excludeall=[], i if not is_src_uri: raise InvalidDependString( _("SRC_URI arrow are only allowed in SRC_URI: token %s") % (pos+1,)) - if eapi is None or not eapi_has_src_uri_arrows(eapi): + if not eapi_attrs.src_uri_arrows: raise InvalidDependString( _("SRC_URI arrow not allowed in EAPI %s: token %s") % (eapi, pos+1)) need_simple_token = True @@ -640,30 +805,9 @@ def flatten(mylist): newlist.append(x) return newlist - -_usedep_re = { - "0": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), - "4-python": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@.-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), -} - -def _get_usedep_re(eapi): - """ - When eapi is None then validation is not as strict, since we want the - same to work for multiple EAPIs that may have slightly different rules. - @param eapi: The EAPI - @type eapi: String or None - @rtype: regular expression object - @return: A regular expression object that matches valid USE deps for the - given eapi. - """ - if eapi in (None, "4-python",): - return _usedep_re["4-python"] - else: - return _usedep_re["0"] - class _use_dep(object): - __slots__ = ("__weakref__", "eapi", "conditional", "missing_enabled", "missing_disabled", + __slots__ = ("_eapi_attrs", "conditional", "missing_enabled", "missing_disabled", "disabled", "enabled", "tokens", "required") class _conditionals_class(object): @@ -689,10 +833,10 @@ class _use_dep(object): 'not_equal': '!%s=', } - def __init__(self, use, eapi, enabled_flags=None, disabled_flags=None, missing_enabled=None, \ + def __init__(self, use, eapi_attrs, enabled_flags=None, disabled_flags=None, missing_enabled=None, missing_disabled=None, conditional=None, required=None): - self.eapi = eapi + self._eapi_attrs = eapi_attrs if enabled_flags is not None: #A shortcut for the classe's own methods. @@ -721,7 +865,7 @@ class _use_dep(object): no_default = set() conditional = {} - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in use: m = usedep_re.match(x) @@ -789,6 +933,14 @@ class _use_dep(object): return "" return "[%s]" % (",".join(self.tokens),) + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__(), + encoding=_encodings['content'], errors='backslashreplace') + def __repr__(self): return "portage.dep._use_dep(%s)" % repr(self.tokens) @@ -824,7 +976,7 @@ class _use_dep(object): disabled_flags = set(self.disabled) tokens = [] - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -860,7 +1012,7 @@ class _use_dep(object): else: tokens.append(x) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=self.missing_enabled, missing_disabled=self.missing_disabled, required=self.required) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -882,7 +1034,7 @@ class _use_dep(object): def validate_flag(flag): return is_valid_flag(flag) or flag in all_defaults - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -974,7 +1126,7 @@ class _use_dep(object): tokens.append(x) conditional.setdefault("disabled", set()).add(flag) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=self.missing_enabled, missing_disabled=self.missing_disabled, \ conditional=conditional, required=self.required) @@ -994,7 +1146,7 @@ class _use_dep(object): missing_disabled = self.missing_disabled tokens = [] - usedep_re = _get_usedep_re(self.eapi) + usedep_re = _get_usedep_re(self._eapi_attrs) for x in self.tokens: m = usedep_re.match(x) @@ -1030,15 +1182,10 @@ class _use_dep(object): else: tokens.append(x) - return _use_dep(tokens, self.eapi, enabled_flags=enabled_flags, disabled_flags=disabled_flags, \ + return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=missing_enabled, missing_disabled=missing_disabled, required=self.required) -if sys.hexversion < 0x3000000: - _atom_base = unicode -else: - _atom_base = str - -class Atom(_atom_base): +class Atom(_unicode): """ For compatibility with existing atom string manipulation code, this @@ -1057,26 +1204,34 @@ class Atom(_atom_base): def __init__(self, forbid_overlap=False): self.overlap = self._overlap(forbid=forbid_overlap) - def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=False, + def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None): - return _atom_base.__new__(cls, s) + return _unicode.__new__(cls, s) - def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=False, + def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None): if isinstance(s, Atom): # This is an efficiency assertion, to ensure that the Atom # constructor is not called redundantly. raise TypeError(_("Expected %s, got %s") % \ - (_atom_base, type(s))) + (_unicode, type(s))) - if not isinstance(s, _atom_base): - # Avoid TypeError from _atom_base.__init__ with PyPy. + if not isinstance(s, _unicode): + # Avoid TypeError from _unicode.__init__ with PyPy. s = _unicode_decode(s) - _atom_base.__init__(s) + _unicode.__init__(s) - if eapi_has_repo_deps(eapi): - allow_repo = True + eapi_attrs = _get_eapi_attrs(eapi) + atom_re = _get_atom_re(eapi_attrs) + + self.__dict__['eapi'] = eapi + if eapi is not None: + # Ignore allow_repo when eapi is specified. + allow_repo = eapi_attrs.repo_deps + else: + if allow_repo is None: + allow_repo = True if "!" == s[:1]: blocker = self._blocker(forbid_overlap=("!" == s[1:2])) @@ -1087,59 +1242,101 @@ class Atom(_atom_base): else: blocker = False self.__dict__['blocker'] = blocker - m = _atom_re.match(s) + m = atom_re.match(s) extended_syntax = False + extended_version = None if m is None: if allow_wildcard: - m = _atom_wildcard_re.match(s) + atom_re = _get_atom_wildcard_re(eapi_attrs) + m = atom_re.match(s) if m is None: raise InvalidAtom(self) - op = None gdict = m.groupdict() - cpv = cp = gdict['simple'] + if m.group('star') is not None: + op = '=*' + base = atom_re.groupindex['star'] + cp = m.group(base + 1) + cpv = m.group('star')[1:] + extended_version = m.group(base + 4) + else: + op = None + cpv = cp = m.group('simple') if cpv.find("**") != -1: raise InvalidAtom(self) - slot = gdict['slot'] - repo = gdict['repo'] + slot = m.group('slot') + repo = m.group('repo') use_str = None extended_syntax = True else: raise InvalidAtom(self) elif m.group('op') is not None: - base = _atom_re.groupindex['op'] + base = atom_re.groupindex['op'] op = m.group(base + 1) cpv = m.group(base + 2) cp = m.group(base + 3) - slot = m.group(_atom_re.groups - 2) - repo = m.group(_atom_re.groups - 1) - use_str = m.group(_atom_re.groups) + slot = m.group(atom_re.groups - 2) + repo = m.group(atom_re.groups - 1) + use_str = m.group(atom_re.groups) if m.group(base + 4) is not None: raise InvalidAtom(self) elif m.group('star') is not None: - base = _atom_re.groupindex['star'] + base = atom_re.groupindex['star'] op = '=*' cpv = m.group(base + 1) cp = m.group(base + 2) - slot = m.group(_atom_re.groups - 2) - repo = m.group(_atom_re.groups - 1) - use_str = m.group(_atom_re.groups) + slot = m.group(atom_re.groups - 2) + repo = m.group(atom_re.groups - 1) + use_str = m.group(atom_re.groups) if m.group(base + 3) is not None: raise InvalidAtom(self) elif m.group('simple') is not None: op = None - cpv = cp = m.group(_atom_re.groupindex['simple'] + 1) - slot = m.group(_atom_re.groups - 2) - repo = m.group(_atom_re.groups - 1) - use_str = m.group(_atom_re.groups) - if m.group(_atom_re.groupindex['simple'] + 2) is not None: + cpv = cp = m.group(atom_re.groupindex['simple'] + 1) + slot = m.group(atom_re.groups - 2) + repo = m.group(atom_re.groups - 1) + use_str = m.group(atom_re.groups) + if m.group(atom_re.groupindex['simple'] + 2) is not None: raise InvalidAtom(self) else: raise AssertionError(_("required group not found in atom: '%s'") % self) self.__dict__['cp'] = cp - self.__dict__['cpv'] = cpv + try: + self.__dict__['cpv'] = _pkg_str(cpv) + self.__dict__['version'] = self.cpv.version + except InvalidData: + # plain cp, wildcard, or something + self.__dict__['cpv'] = cpv + self.__dict__['version'] = extended_version self.__dict__['repo'] = repo - self.__dict__['slot'] = slot + if slot is None: + self.__dict__['slot'] = None + self.__dict__['slot_abi'] = None + self.__dict__['slot_abi_op'] = None + else: + slot_re = _get_slot_dep_re(eapi_attrs) + slot_match = slot_re.match(slot) + if slot_match is None: + raise InvalidAtom(self) + if eapi_attrs.slot_abi: + self.__dict__['slot'] = slot_match.group(1) + slot_abi = slot_match.group(2) + if slot_abi is not None: + slot_abi = slot_abi.lstrip("/") + if slot_abi in ("*", "="): + self.__dict__['slot_abi'] = None + self.__dict__['slot_abi_op'] = slot_abi + else: + slot_abi_op = None + if slot_abi is not None and slot_abi[-1:] == "=": + slot_abi_op = slot_abi[-1:] + slot_abi = slot_abi[:-1] + self.__dict__['slot_abi'] = slot_abi + self.__dict__['slot_abi_op'] = slot_abi_op + else: + self.__dict__['slot'] = slot + self.__dict__['slot_abi'] = None + self.__dict__['slot_abi_op'] = None self.__dict__['operator'] = op self.__dict__['extended_syntax'] = extended_syntax @@ -1150,7 +1347,7 @@ class Atom(_atom_base): if _use is not None: use = _use else: - use = _use_dep(use_str[1:-1].split(","), eapi) + use = _use_dep(use_str[1:-1].split(","), eapi_attrs) without_use = Atom(m.group('without_use'), allow_repo=allow_repo) else: use = None @@ -1175,16 +1372,16 @@ class Atom(_atom_base): if not isinstance(eapi, basestring): raise TypeError('expected eapi argument of ' + \ '%s, got %s: %s' % (basestring, type(eapi), eapi,)) - if self.slot and not eapi_has_slot_deps(eapi): + if self.slot and not eapi_attrs.slot_deps: raise InvalidAtom( _("Slot deps are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') if self.use: - if not eapi_has_use_deps(eapi): + if not eapi_attrs.use_deps: raise InvalidAtom( _("Use deps are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') - elif not eapi_has_use_dep_defaults(eapi) and \ + elif not eapi_attrs.use_dep_defaults and \ (self.use.missing_enabled or self.use.missing_disabled): raise InvalidAtom( _("Use dep defaults are not allowed in EAPI %s: '%s'") \ @@ -1207,12 +1404,21 @@ class Atom(_atom_base): "conditional '%s' in atom '%s' is not in IUSE") \ % (flag, conditional_str % flag, self) raise InvalidAtom(msg, category='IUSE.missing') - if self.blocker and self.blocker.overlap.forbid and not eapi_has_strong_blocks(eapi): + if self.blocker and self.blocker.overlap.forbid and not eapi_attrs.strong_blocks: raise InvalidAtom( _("Strong blocks are not allowed in EAPI %s: '%s'") \ % (eapi, self), category='EAPI.incompatible') @property + def slot_abi_built(self): + """ + Returns True if slot_abi_op == "=" and slot_abi is not None. + NOTE: foo/bar:2= is unbuilt and returns False, whereas foo/bar:2/2= + is built and returns True. + """ + return self.slot_abi_op == "=" and self.slot_abi is not None + + @property def without_repo(self): if self.repo is None: return self @@ -1221,18 +1427,29 @@ class Atom(_atom_base): @property def without_slot(self): - if self.slot is None: + if self.slot is None and self.slot_abi_op is None: return self - return Atom(self.replace(_slot_separator + self.slot, '', 1), + atom = remove_slot(self) + if self.repo is not None: + atom += _repo_separator + self.repo + if self.use is not None: + atom += _unicode(self.use) + return Atom(atom, allow_repo=True, allow_wildcard=True) def with_repo(self, repo): atom = remove_slot(self) - if self.slot is not None: - atom += _slot_separator + self.slot + if self.slot is not None or self.slot_abi_op is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.slot_abi is not None: + atom += "/%s" % self.slot_abi + if self.slot_abi_op is not None: + atom += self.slot_abi_op atom += _repo_separator + repo if self.use is not None: - atom += str(self.use) + atom += _unicode(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def with_slot(self, slot): @@ -1240,7 +1457,7 @@ class Atom(_atom_base): if self.repo is not None: atom += _repo_separator + self.repo if self.use is not None: - atom += str(self.use) + atom += _unicode(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def __setattr__(self, name, value): @@ -1289,10 +1506,16 @@ class Atom(_atom_base): if not (self.use and self.use.conditional): return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_abi_op is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.slot_abi is not None: + atom += "/%s" % self.slot_abi + if self.slot_abi_op is not None: + atom += self.slot_abi_op use_dep = self.use.evaluate_conditionals(use) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -1311,20 +1534,32 @@ class Atom(_atom_base): if not self.use: return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_abi_op is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.slot_abi is not None: + atom += "/%s" % self.slot_abi + if self.slot_abi_op is not None: + atom += self.slot_abi_op use_dep = self.use.violated_conditionals(other_use, is_valid_flag, parent_use) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def _eval_qa_conditionals(self, use_mask, use_force): if not (self.use and self.use.conditional): return self atom = remove_slot(self) - if self.slot: - atom += ":%s" % self.slot + if self.slot is not None or self.slot_abi_op is not None: + atom += _slot_separator + if self.slot is not None: + atom += self.slot + if self.slot_abi is not None: + atom += "/%s" % self.slot_abi + if self.slot_abi_op is not None: + atom += self.slot_abi_op use_dep = self.use._eval_qa_conditionals(use_mask, use_force) - atom += str(use_dep) + atom += _unicode(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def __copy__(self): @@ -1626,52 +1861,8 @@ def dep_getusedeps( depend ): open_bracket = depend.find( '[', open_bracket+1 ) return tuple(use_list) -# \w is [a-zA-Z0-9_] - -# 2.1.3 A slot name may contain any of the characters [A-Za-z0-9+_.-]. -# It must not begin with a hyphen or a dot. -_slot_separator = ":" -_slot = r'([\w+][\w+.-]*)' -_slot_re = re.compile('^' + _slot + '$', re.VERBOSE) - -_use = r'\[.*\]' -_op = r'([=~]|[><]=?)' -_repo_separator = "::" -_repo_name = r'[\w][\w-]*' -_repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' - -_atom_re = re.compile('^(?P<without_use>(?:' + - '(?P<op>' + _op + _cpv + ')|' + - '(?P<star>=' + _cpv + r'\*)|' + - '(?P<simple>' + _cp + '))' + - '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE) - -_extended_cat = r'[\w+*][\w+.*-]*' -_extended_pkg = r'[\w+*][\w+*-]*?' - -_atom_wildcard_re = re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$') - -_useflag_re = { - "0": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@-]*$'), - "4-python": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@.-]*$'), -} - -def _get_useflag_re(eapi): - """ - When eapi is None then validation is not as strict, since we want the - same to work for multiple EAPIs that may have slightly different rules. - @param eapi: The EAPI - @type eapi: String or None - @rtype: regular expression object - @return: A regular expression object that matches valid USE flags for the - given eapi. - """ - if eapi in (None, "4-python",): - return _useflag_re["4-python"] - else: - return _useflag_re["0"] - -def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=False): +def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, + allow_repo=False, eapi=None): """ Check to see if a depend atom is valid @@ -1688,9 +1879,15 @@ def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=Fal 1) False if the atom is invalid 2) True if the atom is valid """ + + if eapi is not None and isinstance(atom, Atom) and atom.eapi != eapi: + # We'll construct a new atom with the given eapi. + atom = _unicode(atom) + try: if not isinstance(atom, Atom): - atom = Atom(atom, allow_wildcard=allow_wildcard, allow_repo=allow_repo) + atom = Atom(atom, allow_wildcard=allow_wildcard, + allow_repo=allow_repo, eapi=eapi) if not allow_blockers and atom.blocker: return False return True @@ -1816,19 +2013,23 @@ def best_match_to_list(mypkg, mylist): """ operator_values = {'=':6, '~':5, '=*':4, '>':2, '<':2, '>=':2, '<=':2, None:1} - maxvalue = -2 + maxvalue = -99 bestm = None mypkg_cpv = None for x in match_to_list(mypkg, mylist): if x.extended_syntax: - if dep_getslot(x) is not None: + if x.operator == '=*': if maxvalue < 0: maxvalue = 0 bestm = x - else: + elif x.slot is not None: if maxvalue < -1: maxvalue = -1 bestm = x + else: + if maxvalue < -2: + maxvalue = -2 + bestm = x continue if dep_getslot(x) is not None: if maxvalue < 3: @@ -1842,9 +2043,10 @@ def best_match_to_list(mypkg, mylist): # For >, <, >=, and <=, the one with the version # closest to mypkg is the best match. if mypkg_cpv is None: - mypkg_cpv = getattr(mypkg, "cpv", None) - if mypkg_cpv is None: - mypkg_cpv = remove_slot(mypkg) + try: + mypkg_cpv = mypkg.cpv + except AttributeError: + mypkg_cpv = _pkg_str(remove_slot(mypkg)) if bestm.cpv == mypkg_cpv or bestm.cpv == x.cpv: pass elif x.cpv == mypkg_cpv: @@ -1852,11 +2054,8 @@ def best_match_to_list(mypkg, mylist): else: # Sort the cpvs to find the one closest to mypkg_cpv cpv_list = [bestm.cpv, mypkg_cpv, x.cpv] - ver_map = {} - for cpv in cpv_list: - ver_map[cpv] = '-'.join(catpkgsplit(cpv)[2:]) def cmp_cpv(cpv1, cpv2): - return vercmp(ver_map[cpv1], ver_map[cpv2]) + return vercmp(cpv1.version, cpv2.version) cpv_list.sort(key=cmp_sort_key(cmp_cpv)) if cpv_list[0] is mypkg_cpv or cpv_list[-1] is mypkg_cpv: if cpv_list[1] is x.cpv: @@ -1882,7 +2081,6 @@ def match_from_list(mydep, candidate_list): if not candidate_list: return [] - from portage.util import writemsg if "!" == mydep[:1]: if "!" == mydep[1:2]: mydep = mydep[2:] @@ -1915,7 +2113,39 @@ def match_from_list(mydep, candidate_list): mylist = [] - if operator is None: + if mydep.extended_syntax: + + for x in candidate_list: + cp = getattr(x, "cp", None) + if cp is None: + mysplit = catpkgsplit(remove_slot(x)) + if mysplit is not None: + cp = mysplit[0] + '/' + mysplit[1] + + if cp is None: + continue + + if cp == mycpv or extended_cp_match(mydep.cp, cp): + mylist.append(x) + + if mylist and mydep.operator == "=*": + + candidate_list = mylist + mylist = [] + # Currently, only \*\d+\* is supported. + ver = mydep.version[1:-1] + + for x in candidate_list: + x_ver = getattr(x, "version", None) + if x_ver is None: + xs = catpkgsplit(remove_slot(x)) + if xs is None: + continue + x_ver = "-".join(xs[-2:]) + if ver in x_ver: + mylist.append(x) + + elif operator is None: for x in candidate_list: cp = getattr(x, "cp", None) if cp is None: @@ -1926,8 +2156,7 @@ def match_from_list(mydep, candidate_list): if cp is None: continue - if cp == mycpv or (mydep.extended_syntax and \ - extended_cp_match(mydep.cp, cp)): + if cp == mydep.cp: mylist.append(x) elif operator == "=": # Exact match @@ -1947,7 +2176,7 @@ def match_from_list(mydep, candidate_list): myver = mysplit[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver - mycpv = mysplit[0]+"/"+mysplit[1]+"-"+myver + mycpv_cmp = mysplit[0]+"/"+mysplit[1]+"-"+myver for x in candidate_list: xs = getattr(x, "cpv_split", None) if xs is None: @@ -1956,7 +2185,7 @@ def match_from_list(mydep, candidate_list): if not myver or not myver[0].isdigit(): myver = "0"+myver xcpv = xs[0]+"/"+xs[1]+"-"+myver - if xcpv.startswith(mycpv): + if xcpv.startswith(mycpv_cmp): mylist.append(x) elif operator == "~": # version, any revision, match @@ -1973,15 +2202,19 @@ def match_from_list(mydep, candidate_list): mylist.append(x) elif operator in [">", ">=", "<", "<="]: - mysplit = ["%s/%s" % (cat, pkg), ver, rev] for x in candidate_list: - xs = getattr(x, "cpv_split", None) - if xs is None: - xs = catpkgsplit(remove_slot(x)) - xcat, xpkg, xver, xrev = xs - xs = ["%s/%s" % (xcat, xpkg), xver, xrev] + if hasattr(x, 'cp'): + pkg = x + else: + try: + pkg = _pkg_str(remove_slot(x)) + except InvalidData: + continue + + if pkg.cp != mydep.cp: + continue try: - result = pkgcmp(xs, mysplit) + result = vercmp(pkg.version, mydep.version) except ValueError: # pkgcmp may return ValueError during int() conversion writemsg(_("\nInvalid package name: %s\n") % x, noiselevel=-1) raise @@ -2004,16 +2237,33 @@ def match_from_list(mydep, candidate_list): else: raise KeyError(_("Unknown operator: %s") % mydep) - if slot is not None and not mydep.extended_syntax: + if mydep.slot is not None: candidate_list = mylist mylist = [] for x in candidate_list: - xslot = getattr(x, "slot", False) - if xslot is False: + x_pkg = None + try: + x.cpv + except AttributeError: xslot = dep_getslot(x) - if xslot is not None and xslot != slot: - continue - mylist.append(x) + if xslot is not None: + try: + x_pkg = _pkg_str(remove_slot(x), slot=xslot) + except InvalidData: + continue + else: + x_pkg = x + + if x_pkg is None: + mylist.append(x) + else: + try: + x_pkg.slot + except AttributeError: + mylist.append(x) + else: + if _match_slot(mydep, x_pkg): + mylist.append(x) if mydep.unevaluated_atom.use: candidate_list = mylist @@ -2032,21 +2282,19 @@ def match_from_list(mydep, candidate_list): missing_disabled = mydep.use.missing_disabled.difference(x.iuse.all) if mydep.use.enabled: - if mydep.use.enabled.intersection(missing_disabled): + if any(f in mydep.use.enabled for f in missing_disabled): continue need_enabled = mydep.use.enabled.difference(use.enabled) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: + if any(f not in missing_enabled for f in need_enabled): continue if mydep.use.disabled: - if mydep.use.disabled.intersection(missing_enabled): + if any(f in mydep.use.disabled for f in missing_enabled): continue need_disabled = mydep.use.disabled.intersection(use.enabled) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: + if any(f not in missing_disabled for f in need_disabled): continue mylist.append(x) @@ -2058,7 +2306,8 @@ def match_from_list(mydep, candidate_list): repo = getattr(x, "repo", False) if repo is False: repo = dep_getrepo(x) - if repo is not None and repo != mydep.repo: + if repo is not None and repo != _unknown_repo and \ + repo != mydep.repo: continue mylist.append(x) diff --git a/pym/portage/dep/_slot_abi.py b/pym/portage/dep/_slot_abi.py new file mode 100644 index 00000000000..7c36e52dc4f --- /dev/null +++ b/pym/portage/dep/_slot_abi.py @@ -0,0 +1,92 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.dep import Atom, paren_enclose, use_reduce +from portage.exception import InvalidData + +_dep_keys = ('DEPEND', 'PDEPEND', 'RDEPEND') +_runtime_keys = ('PDEPEND', 'RDEPEND') + +def find_built_slot_abi_atoms(pkg): + atoms = {} + for k in _dep_keys: + atom_list = list(_find_built_slot_abi_op(use_reduce(pkg.metadata[k], + uselist=pkg.use.enabled, eapi=pkg.metadata['EAPI'], + token_class=Atom))) + if atom_list: + atoms[k] = atom_list + return atoms + +def _find_built_slot_abi_op(dep_struct): + for x in dep_struct: + if isinstance(x, list): + for atom in _find_built_slot_abi_op(x): + yield atom + elif isinstance(x, Atom) and x.slot_abi_built: + yield x + +def ignore_built_slot_abi_deps(dep_struct): + for i, x in enumerate(dep_struct): + if isinstance(x, list): + ignore_built_slot_abi_deps(x) + elif isinstance(x, Atom) and x.slot_abi_built: + # There's no way of knowing here whether the SLOT + # part of the SLOT/ABI pair should be kept, so we + # ignore both parts. + dep_struct[i] = x.without_slot + +def evaluate_slot_abi_equal_deps(settings, use, trees): + + metadata = settings.configdict['pkg'] + eapi = metadata['EAPI'] + running_vardb = trees[trees._running_eroot]["vartree"].dbapi + target_vardb = trees[trees._target_eroot]["vartree"].dbapi + vardbs = [target_vardb] + deps = {} + for k in _dep_keys: + deps[k] = use_reduce(metadata[k], + uselist=use, eapi=eapi, token_class=Atom) + + for k in _runtime_keys: + _eval_deps(deps[k], vardbs) + + if running_vardb is not target_vardb: + vardbs.append(running_vardb) + + _eval_deps(deps["DEPEND"], vardbs) + + result = {} + for k, v in deps.items(): + result[k] = paren_enclose(v) + + return result + +def _eval_deps(dep_struct, vardbs): + for i, x in enumerate(dep_struct): + if isinstance(x, list): + _eval_deps(x, vardbs) + elif isinstance(x, Atom) and x.slot_abi_op == "=": + for vardb in vardbs: + best_version = vardb.match(x) + if best_version: + best_version = best_version[-1] + try: + best_version = \ + vardb._pkg_str(best_version, None) + except (KeyError, InvalidData): + pass + else: + slot_part = "%s/%s=" % \ + (best_version.slot, best_version.slot_abi) + x = x.with_slot(slot_part) + dep_struct[i] = x + break + else: + # this dep could not be resolved, so remove the operator + # (user may be using package.provided and managing rebuilds + # manually) + if x.slot: + x = x.with_slot(x.slot) + else: + x = x.without_slot + dep_struct[i] = x diff --git a/pym/portage/dep/dep_check.py b/pym/portage/dep/dep_check.py index 0866673c12e..d575ab3bca1 100644 --- a/pym/portage/dep/dep_check.py +++ b/pym/portage/dep/dep_check.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['dep_check', 'dep_eval', 'dep_wordreduce', 'dep_zapdeps'] @@ -11,7 +11,7 @@ from portage.dep import Atom, match_from_list, use_reduce from portage.exception import InvalidDependString, ParseError from portage.localization import _ from portage.util import writemsg, writemsg_level -from portage.versions import catpkgsplit, cpv_getkey, pkgcmp +from portage.versions import vercmp, _pkg_str def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", trees=None, use_mask=None, use_force=None, **kwargs): @@ -39,14 +39,12 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", parent = mytrees.get("parent") virt_parent = mytrees.get("virt_parent") graph_parent = None - eapi = None if parent is not None: if virt_parent is not None: graph_parent = virt_parent parent = virt_parent else: graph_parent = parent - eapi = parent.metadata["EAPI"] repoman = not mysettings.local_config if kwargs["use_binaries"]: portdb = trees[myroot]["bintree"].dbapi @@ -352,8 +350,14 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg = mydbapi.match(atom.without_use) if avail_pkg: avail_pkg = avail_pkg[-1] # highest (ascending order) - avail_slot = Atom("%s:%s" % (atom.cp, - mydbapi.aux_get(avail_pkg, ["SLOT"])[0])) + try: + slot = avail_pkg.slot + except AttributeError: + eapi, slot, repo = mydbapi.aux_get(avail_pkg, + ["EAPI", "SLOT", "repository"]) + avail_pkg = _pkg_str(avail_pkg, eapi=eapi, + slot=slot, repo=repo) + avail_slot = Atom("%s:%s" % (atom.cp, slot)) if not avail_pkg: all_available = False all_use_satisfied = False @@ -368,16 +372,19 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg_use = avail_pkg_use[-1] if avail_pkg_use != avail_pkg: avail_pkg = avail_pkg_use - avail_slot = Atom("%s:%s" % (atom.cp, - mydbapi.aux_get(avail_pkg, ["SLOT"])[0])) + try: + slot = avail_pkg.slot + except AttributeError: + eapi, slot, repo = mydbapi.aux_get(avail_pkg, + ["EAPI", "SLOT", "repository"]) + avail_pkg = _pkg_str(avail_pkg, + eapi=eapi, slot=slot, repo=repo) slot_map[avail_slot] = avail_pkg - pkg_cp = cpv_getkey(avail_pkg) - highest_cpv = cp_map.get(pkg_cp) + highest_cpv = cp_map.get(avail_pkg.cp) if highest_cpv is None or \ - pkgcmp(catpkgsplit(avail_pkg)[1:], - catpkgsplit(highest_cpv)[1:]) > 0: - cp_map[pkg_cp] = avail_pkg + vercmp(avail_pkg.version, highest_cpv.version) > 0: + cp_map[avail_pkg.cp] = avail_pkg this_choice = (atoms, slot_map, cp_map, all_available) if all_available: @@ -515,8 +522,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for cp in intersecting_cps: version_1 = cp_map_1[cp] version_2 = cp_map_2[cp] - difference = pkgcmp(catpkgsplit(version_1)[1:], - catpkgsplit(version_2)[1:]) + difference = vercmp(version_1.version, version_2.version) if difference != 0: if difference > 0: has_upgrade = True @@ -605,12 +611,15 @@ def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, if not current_parent.installed: eapi = current_parent.metadata['EAPI'] - try: - mysplit = use_reduce(depstring, uselist=myusesplit, masklist=mymasks, \ - matchall=(use=="all"), excludeall=useforce, opconvert=True, \ - token_class=Atom, eapi=eapi) - except InvalidDependString as e: - return [0, _unicode_decode("%s") % (e,)] + if isinstance(depstring, list): + mysplit = depstring + else: + try: + mysplit = use_reduce(depstring, uselist=myusesplit, + masklist=mymasks, matchall=(use=="all"), excludeall=useforce, + opconvert=True, token_class=Atom, eapi=eapi) + except InvalidDependString as e: + return [0, _unicode_decode("%s") % (e,)] if mysplit == []: #dependencies were reduced to nothing diff --git a/pym/portage/dispatch_conf.py b/pym/portage/dispatch_conf.py index c81153ae197..4c68dfc7b8d 100644 --- a/pym/portage/dispatch_conf.py +++ b/pym/portage/dispatch_conf.py @@ -13,7 +13,7 @@ import os, shutil, subprocess, sys import portage from portage.env.loaders import KeyValuePairFileLoader from portage.localization import _ -from portage.util import varexpand +from portage.util import shlex_split, varexpand RCS_BRANCH = '1.1.1' RCS_LOCK = 'rcs -ko -M -l' @@ -30,8 +30,12 @@ def diffstatusoutput(cmd, file1, file2): """ # Use Popen to emulate getstatusoutput(), since getstatusoutput() may # raise a UnicodeDecodeError which makes the output inaccessible. - proc = subprocess.Popen(cmd % (file1, file2), - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + args = shlex_split(cmd % (file1, file2)) + if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: + # Python 3.1 does not support bytes in Popen args. + args = [portage._unicode_encode(x, errors='strict') for x in args] + proc = subprocess.Popen(args, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = portage._unicode_decode(proc.communicate()[0]) if output and output[-1] == "\n": # getstatusoutput strips one newline diff --git a/pym/portage/eapi.py b/pym/portage/eapi.py index c3c4f2d22b1..8b03f830e8b 100644 --- a/pym/portage/eapi.py +++ b/pym/portage/eapi.py @@ -1,12 +1,17 @@ # Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import collections + def eapi_has_iuse_defaults(eapi): return eapi != "0" def eapi_has_slot_deps(eapi): return eapi != "0" +def eapi_has_slot_abi(eapi): + return eapi in ("4-slot-abi",) + def eapi_has_src_uri_arrows(eapi): return eapi not in ("0", "1") @@ -54,3 +59,42 @@ def eapi_has_use_dep_defaults(eapi): def eapi_has_repo_deps(eapi): return eapi in ("4-python",) + +def eapi_allows_dots_in_PN(eapi): + return eapi in ("4-python",) + +def eapi_allows_dots_in_use_flags(eapi): + return eapi in ("4-python",) + +_eapi_attrs = collections.namedtuple('_eapi_attrs', + 'dots_in_PN dots_in_use_flags iuse_defaults ' + 'repo_deps required_use slot_abi slot_deps ' + 'src_uri_arrows strong_blocks use_deps use_dep_defaults') + +_eapi_attrs_cache = {} + +def _get_eapi_attrs(eapi): + """ + When eapi is None then validation is not as strict, since we want the + same to work for multiple EAPIs that may have slightly different rules. + """ + eapi_attrs = _eapi_attrs_cache.get(eapi) + if eapi_attrs is not None: + return eapi_attrs + + eapi_attrs = _eapi_attrs( + dots_in_PN = (eapi is None or eapi_allows_dots_in_PN(eapi)), + dots_in_use_flags = (eapi is None or eapi_allows_dots_in_use_flags(eapi)), + iuse_defaults = (eapi is None or eapi_has_iuse_defaults(eapi)), + repo_deps = (eapi is None or eapi_has_repo_deps(eapi)), + required_use = (eapi is None or eapi_has_required_use(eapi)), + slot_deps = (eapi is None or eapi_has_slot_deps(eapi)), + slot_abi = (eapi is None or eapi_has_slot_abi(eapi)), + src_uri_arrows = (eapi is None or eapi_has_src_uri_arrows(eapi)), + strong_blocks = (eapi is None or eapi_has_strong_blocks(eapi)), + use_deps = (eapi is None or eapi_has_use_deps(eapi)), + use_dep_defaults = (eapi is None or eapi_has_use_dep_defaults(eapi)) + ) + + _eapi_attrs_cache[eapi] = eapi_attrs + return eapi_attrs diff --git a/pym/portage/emaint/__init__.py b/pym/portage/emaint/__init__.py new file mode 100644 index 00000000000..5e0ae700a0f --- /dev/null +++ b/pym/portage/emaint/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program provides checks and maintenance +on a gentoo system. +""" + diff --git a/pym/portage/emaint/defaults.py b/pym/portage/emaint/defaults.py new file mode 100644 index 00000000000..d9d83ffbb11 --- /dev/null +++ b/pym/portage/emaint/defaults.py @@ -0,0 +1,18 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# parser option data +CHECK = {"short": "-c", "long": "--check", + "help": "Check for problems (a default option for most modules)", + 'status': "Checking %s for problems", + 'func': 'check' + } + +FIX = {"short": "-f", "long": "--fix", + "help": "Attempt to fix problems (a default option for most modules)", + 'status': "Attempting to fix %s", + 'func': 'fix' + } + +# parser options +DEFAULT_OPTIONS = {'check': CHECK, 'fix': FIX} diff --git a/pym/portage/emaint/main.py b/pym/portage/emaint/main.py new file mode 100644 index 00000000000..dbc5f18cc26 --- /dev/null +++ b/pym/portage/emaint/main.py @@ -0,0 +1,218 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + + +import sys +import textwrap +from optparse import OptionParser, OptionValueError + + +import portage +from portage import os +from portage.emaint.module import Modules +from portage.emaint.progress import ProgressBar +from portage.emaint.defaults import DEFAULT_OPTIONS + +class OptionItem(object): + """class to hold module OptionParser options data + """ + + def __init__(self, opt, parser): + """ + @type opt: dictionary + @param opt: options parser options + """ + self.parser = parser + self.short = opt['short'] + self.long = opt['long'] + self.help = opt['help'] + self.status = opt['status'] + self.func = opt['func'] + self.action = opt.get('action', "callback") + self.type = opt.get('type', None) + self.dest = opt.get('dest', None) + self.callback = opt.get('callback', self._exclusive) + self.callback_kwargs = opt.get('callback_kwargs', {"var":"action"}) + + + def _exclusive(self, option, *args, **kw): + """Generic check for the 2 default options + """ + var = kw.get("var", None) + if var is None: + raise ValueError("var not specified to exclusive()") + if getattr(self.parser, var, ""): + raise OptionValueError("%s and %s are exclusive options" + % (getattr(self.parser, var), option)) + setattr(self.parser, var, str(option)) + + def check_action(self, action): + """Checks if 'action' is the same as this option + + @type action: string + @param action: the action to compare + @rtype: boolean + """ + if action == self.action: + return True + elif action == '/'.join([self.short, self.long]): + return True + return False + + +def usage(module_controller): + _usage = "usage: emaint [options] COMMAND" + + desc = "The emaint program provides an interface to system health " + \ + "checks and maintenance. See the emaint(1) man page " + \ + "for additional information about the following commands:" + + _usage += "\n\n" + for line in textwrap.wrap(desc, 65): + _usage += "%s\n" % line + _usage += "\nCommands:\n" + _usage += " %s" % "all".ljust(15) + \ + "Perform all supported commands\n" + textwrap.subsequent_indent = ' '.ljust(17) + for mod in module_controller.module_names: + desc = textwrap.wrap(module_controller.get_description(mod), 65) + _usage += " %s%s\n" % (mod.ljust(15), desc[0]) + for d in desc[1:]: + _usage += " %s%s\n" % (' '.ljust(15), d) + return _usage + + +def module_opts(module_controller, module): + _usage = " %s module options:\n" % module + opts = module_controller.get_func_descriptions(module) + if opts == {}: + opts = DEFAULT_OPTIONS + for opt in sorted(opts): + optd = opts[opt] + opto = " %s, %s" %(optd['short'], optd['long']) + _usage += '%s %s\n' % (opto.ljust(15),optd['help']) + _usage += '\n' + return _usage + + +class TaskHandler(object): + """Handles the running of the tasks it is given + """ + + def __init__(self, show_progress_bar=True, verbose=True, callback=None): + self.show_progress_bar = show_progress_bar + self.verbose = verbose + self.callback = callback + self.isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() + self.progress_bar = ProgressBar(self.isatty, title="Emaint", max_desc_length=27) + + + def run_tasks(self, tasks, func, status=None, verbose=True, options=None): + """Runs the module tasks""" + if tasks is None or func is None: + return + for task in tasks: + inst = task() + show_progress = self.show_progress_bar + # check if the function is capable of progressbar + # and possibly override it off + if show_progress and hasattr(inst, 'can_progressbar'): + show_progress = inst.can_progressbar(func) + if show_progress: + self.progress_bar.reset() + self.progress_bar.set_label(func + " " + inst.name()) + onProgress = self.progress_bar.start() + else: + onProgress = None + kwargs = { + 'onProgress': onProgress, + # pass in a copy of the options so a module can not pollute or change + # them for other tasks if there is more to do. + 'options': options.copy() + } + result = getattr(inst, func)(**kwargs) + if self.isatty and show_progress: + # make sure the final progress is displayed + self.progress_bar.display() + print() + self.progress_bar.stop() + if self.callback: + self.callback(result) + + +def print_results(results): + if results: + print() + print("\n".join(results)) + print("\n") + + +def emaint_main(myargv): + + # Similar to emerge, emaint needs a default umask so that created + # files (such as the world file) have sane permissions. + os.umask(0o22) + + module_controller = Modules(namepath="portage.emaint.modules") + module_names = module_controller.module_names[:] + module_names.insert(0, "all") + + + parser = OptionParser(usage=usage(module_controller), version=portage.VERSION) + # add default options + parser_options = [] + for opt in DEFAULT_OPTIONS: + parser_options.append(OptionItem(DEFAULT_OPTIONS[opt], parser)) + for mod in module_names[1:]: + desc = module_controller.get_func_descriptions(mod) + if desc: + for opt in desc: + parser_options.append(OptionItem(desc[opt], parser)) + for opt in parser_options: + parser.add_option(opt.short, opt.long, help=opt.help, action=opt.action, + type=opt.type, dest=opt.dest, + callback=opt.callback, callback_kwargs=opt.callback_kwargs) + + parser.action = None + + (options, args) = parser.parse_args(args=myargv) + #print('options', options, '\nargs', args, '\naction', parser.action) + if len(args) != 1: + parser.error("Incorrect number of arguments") + if args[0] not in module_names: + parser.error("%s target is not a known target" % args[0]) + + if parser.action: + action = parser.action + else: + action = "-c/--check" + long_action = action.split('/')[1].lstrip('-') + #print("DEBUG: action = ", action, long_action) + + if args[0] == "all": + tasks = [] + for m in module_names[1:]: + #print("DEBUG: module: %s, functions: " %(m, str(module_controller.get_functions(m)))) + if long_action in module_controller.get_functions(m): + tasks.append(module_controller.get_class(m)) + elif long_action in module_controller.get_functions(args[0]): + tasks = [module_controller.get_class(args[0] )] + else: + print("\nERROR: module '%s' does not have option '%s'\n" %(args[0], action)) + print(module_opts(module_controller, args[0])) + sys.exit(1) + func = status = None + for opt in parser_options: + if opt.check_action(action): + status = opt.status + func = opt.func + break + + # need to pass the parser options dict to the modules + # so they are available if needed. + task_opts = options.__dict__ + taskmaster = TaskHandler(callback=print_results) + taskmaster.run_tasks(tasks, func, status, options=task_opts) + diff --git a/pym/portage/emaint/module.py b/pym/portage/emaint/module.py new file mode 100644 index 00000000000..64b0c64b51a --- /dev/null +++ b/pym/portage/emaint/module.py @@ -0,0 +1,194 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + +from portage import os +from portage.exception import PortageException +from portage.cache.mappings import ProtectedDict + + +class InvalidModuleName(PortageException): + """An invalid or unknown module name.""" + + +class Module(object): + """Class to define and hold our plug-in module + + @type name: string + @param name: the module name + @type path: the path to the new module + """ + + def __init__(self, name, namepath): + """Some variables initialization""" + self.name = name + self._namepath = namepath + self.kids_names = [] + self.kids = {} + self.initialized = self._initialize() + + def _initialize(self): + """Initialize the plug-in module + + @rtype: boolean + """ + self.valid = False + try: + mod_name = ".".join([self._namepath, self.name]) + self._module = __import__(mod_name, [],[], ["not empty"]) + self.valid = True + except ImportError as e: + print("MODULE; failed import", mod_name, " error was:",e) + return False + self.module_spec = self._module.module_spec + for submodule in self.module_spec['provides']: + kid = self.module_spec['provides'][submodule] + kidname = kid['name'] + kid['module_name'] = '.'.join([mod_name, self.name]) + kid['is_imported'] = False + self.kids[kidname] = kid + self.kids_names.append(kidname) + return True + + def get_class(self, name): + if not name or name not in self.kids_names: + raise InvalidModuleName("Module name '%s' was invalid or not" + %name + "part of the module '%s'" %self.name) + kid = self.kids[name] + if kid['is_imported']: + module = kid['instance'] + else: + try: + module = __import__(kid['module_name'], [],[], ["not empty"]) + kid['instance'] = module + kid['is_imported'] = True + except ImportError: + raise + mod_class = getattr(module, kid['class']) + return mod_class + + +class Modules(object): + """Dynamic modules system for loading and retrieving any of the + installed emaint modules and/or provided class's + + @param path: Optional path to the "modules" directory or + defaults to the directory of this file + '/modules' + @param namepath: Optional python import path to the "modules" directory or + defaults to the directory name of this file + '.modules' + """ + + def __init__(self, path=None, namepath=None): + if path: + self._module_path = path + else: + self._module_path = os.path.join(( + os.path.dirname(os.path.realpath(__file__))), "modules") + if namepath: + self._namepath = namepath + else: + self._namepath = '.'.join(os.path.dirname( + os.path.realpath(__file__)), "modules") + self._modules = self._get_all_modules() + self.modules = ProtectedDict(self._modules) + self.module_names = sorted(self._modules) + #self.modules = {} + #for mod in self.module_names: + #self.module[mod] = LazyLoad( + + def _get_all_modules(self): + """scans the emaint modules dir for loadable modules + + @rtype: dictionary of module_plugins + """ + module_dir = self._module_path + importables = [] + names = os.listdir(module_dir) + for entry in names: + # skip any __init__ or __pycache__ files or directories + if entry.startswith('__'): + continue + try: + # test for statinfo to ensure it should a real module + # it will bail if it errors + os.lstat(os.path.join(module_dir, entry, '__init__.py')) + importables.append(entry) + except EnvironmentError: + pass + kids = {} + for entry in importables: + new_module = Module(entry, self._namepath) + for module_name in new_module.kids: + kid = new_module.kids[module_name] + kid['parent'] = new_module + kids[kid['name']] = kid + return kids + + def get_module_names(self): + """Convienence function to return the list of installed modules + available + + @rtype: list + @return: the installed module names available + """ + return self.module_names + + def get_class(self, modname): + """Retrieves a module class desired + + @type modname: string + @param modname: the module class name + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['parent'].get_class(modname) + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_description(self, modname): + """Retrieves the module class decription + + @type modname: string + @param modname: the module class name + @type string + @return: the modules class decription + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['description'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_functions(self, modname): + """Retrieves the module class exported function names + + @type modname: string + @param modname: the module class name + @type list + @return: the modules class exported function names + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['functions'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_func_descriptions(self, modname): + """Retrieves the module class exported functions descriptions + + @type modname: string + @param modname: the module class name + @type dictionary + @return: the modules class exported functions descriptions + """ + if modname and modname in self.module_names: + desc = self._modules[modname]['func_desc'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return desc diff --git a/pym/portage/emaint/modules/__init__.py b/pym/portage/emaint/modules/__init__.py new file mode 100644 index 00000000000..35674e34223 --- /dev/null +++ b/pym/portage/emaint/modules/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program plug-in module provides an automatic method +of adding/removing modules to perform checks and maintenance +on a gentoo system. +""" diff --git a/pym/portage/emaint/modules/binhost/__init__.py b/pym/portage/emaint/modules/binhost/__init__.py new file mode 100644 index 00000000000..1a61af42bf6 --- /dev/null +++ b/pym/portage/emaint/modules/binhost/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'The emaint program module provides checks and maintenancefor: + Scanning, checking and fixing problems in the world file. +""" + + +module_spec = { + 'name': 'binhost', + 'description': "Provides functions to scan, check and " + \ + "Generate a metadata index for binary packages", + 'provides':{ + 'module1': { + 'name': "binhost", + 'class': "BinhostHandler", + 'description': "Generate a metadata index for binary packages", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/binhost/binhost.py b/pym/portage/emaint/modules/binhost/binhost.py new file mode 100644 index 00000000000..b540d76860e --- /dev/null +++ b/pym/portage/emaint/modules/binhost/binhost.py @@ -0,0 +1,167 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import stat + +import portage +from portage import os +from portage.util import writemsg + +import sys +if sys.hexversion >= 0x3000000: + long = int + +class BinhostHandler(object): + + short_desc = "Generate a metadata index for binary packages" + + def name(): + return "binhost" + name = staticmethod(name) + + def __init__(self): + eroot = portage.settings['EROOT'] + self._bintree = portage.db[eroot]["bintree"] + self._bintree.populate() + self._pkgindex_file = self._bintree._pkgindex_file + self._pkgindex = self._bintree._load_pkgindex() + + def _need_update(self, cpv, data): + + if "MD5" not in data: + return True + + size = data.get("SIZE") + if size is None: + return True + + mtime = data.get("MTIME") + if mtime is None: + return True + + pkg_path = self._bintree.getname(cpv) + try: + s = os.lstat(pkg_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + # We can't update the index for this one because + # it disappeared. + return False + + try: + if long(mtime) != s[stat.ST_MTIME]: + return True + if long(size) != long(s.st_size): + return True + except ValueError: + return True + + return False + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + missing = [] + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + if onProgress: + onProgress(maxval, i+1) + errors = ["'%s' is not in Packages" % cpv for cpv in missing] + stale = set(metadata).difference(cpv_all) + for cpv in stale: + errors.append("'%s' is not in the repository" % cpv) + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + bintree = self._bintree + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + missing = [] + maxval = 0 + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + + stale = set(metadata).difference(cpv_all) + if missing or stale: + from portage import locks + pkgindex_lock = locks.lockfile( + self._pkgindex_file, wantnewlockfile=1) + try: + # Repopulate with lock held. + bintree._populate() + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + + pkgindex = bintree._load_pkgindex() + self._pkgindex = pkgindex + + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + # Recount missing packages, with lock held. + del missing[:] + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + + maxval = len(missing) + for i, cpv in enumerate(missing): + try: + metadata[cpv] = bintree._pkgindex_entry(cpv) + except portage.exception.InvalidDependString: + writemsg("!!! Invalid binary package: '%s'\n" % \ + bintree.getname(cpv), noiselevel=-1) + + if onProgress: + onProgress(maxval, i+1) + + for cpv in set(metadata).difference( + self._bintree.dbapi.cpv_all()): + del metadata[cpv] + + # We've updated the pkgindex, so set it to + # repopulate when necessary. + bintree.populated = False + + del pkgindex.packages[:] + pkgindex.packages.extend(metadata.values()) + from portage.util import atomic_ofstream + f = atomic_ofstream(self._pkgindex_file) + try: + self._pkgindex.write(f) + finally: + f.close() + finally: + locks.unlockfile(pkgindex_lock) + + if onProgress: + if maxval == 0: + maxval = 1 + onProgress(maxval, maxval) + return None diff --git a/pym/portage/emaint/modules/config/__init__.py b/pym/portage/emaint/modules/config/__init__.py new file mode 100644 index 00000000000..22abb07b152 --- /dev/null +++ b/pym/portage/emaint/modules/config/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the emerge config tracker list +""" + + +module_spec = { + 'name': 'config', + 'description': "Provides functions to scan, check for and fix no " +\ + "longer installed config files in emerge's tracker file", + 'provides':{ + 'module1': { + 'name': "cleanconfmem", + 'class': "CleanConfig", + 'description': "Discard no longer installed config tracker entries", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/config/config.py b/pym/portage/emaint/modules/config/config.py new file mode 100644 index 00000000000..a80d87d296b --- /dev/null +++ b/pym/portage/emaint/modules/config/config.py @@ -0,0 +1,101 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.const import PRIVATE_PATH +from portage.checksum import perform_md5 + + +class CleanConfig(object): + + short_desc = "Discard any no longer installed configs from emerge's tracker list" + + def __init__(self): + self.target = os.path.join(portage.settings["EROOT"], PRIVATE_PATH, 'config') + + def name(): + return "cleanconfmem" + name = staticmethod(name) + + def load_configlist(self): + + configs = {} + with open(self.target, 'r') as configfile: + lines = configfile.readlines() + for line in lines: + ls = line.split() + configs[ls[0]] = ls[1] + return configs + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + configs = self.load_configlist() + messages = [] + chksums = [] + maxval = len(configs) + if onProgress: + onProgress(maxval, 0) + i = 0 + keys = sorted(configs) + for config in keys: + if os.path.exists(config): + md5sumactual = perform_md5(config) + if md5sumactual != configs[config]: + chksums.append(" %s" % config) + else: + messages.append(" %s" % config) + if onProgress: + onProgress(maxval, i+1) + i += 1 + return self._format_output(messages, chksums) + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + configs = self.load_configlist() + messages = [] + chksums = [] + maxval = len(configs) + if onProgress: + onProgress(maxval, 0) + i = 0 + keys = sorted(configs) + for config in keys: + if os.path.exists(config): + md5sumactual = perform_md5(config) + if md5sumactual != configs[config]: + chksums.append(" %s" % config) + configs.pop(config) + else: + configs.pop(config) + messages.append(" %s" % config) + if onProgress: + onProgress(maxval, i+1) + i += 1 + lines = [] + keys = sorted(configs) + for key in keys: + line = ' '.join([key, configs[key]]) + lines.append(line) + lines.append('') + with open(self.target, 'w') as configfile: + configfile.write('\n'.join(lines)) + return self._format_output(messages, chksums, True) + + def _format_output(self, messages=[], chksums=[], cleaned=False): + output = [] + if messages: + output.append('Not Installed:') + output += messages + tot = '------------------------------------\n Total %i Not installed' + if cleaned: + tot += ' ...Cleaned' + output.append(tot % len(messages)) + if chksums: + output.append('\nChecksums did not match:') + output += chksums + tot = '------------------------------------\n Total %i Checksums did not match' + if cleaned: + tot += ' ...Cleaned' + output.append(tot % len(chksums)) + return output diff --git a/pym/portage/emaint/modules/logs/__init__.py b/pym/portage/emaint/modules/logs/__init__.py new file mode 100644 index 00000000000..005b608a637 --- /dev/null +++ b/pym/portage/emaint/modules/logs/__init__.py @@ -0,0 +1,51 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the PORT_LOGDIR logs +""" + + +module_spec = { + 'name': 'logs', + 'description': "Provides functions to scan, check and clean old logs " +\ + "in the PORT_LOGDIR", + 'provides':{ + 'module1': { + 'name': "logs", + 'class': "CleanLogs", + 'description': "Clean out old logs from the PORT_LOGDIR", + 'functions': ['check','clean'], + 'func_desc': { + 'clean': { + "short": "-C", "long": "--clean", + "help": "Cleans out logs more than 7 days old (cleanlogs only)" + \ + " modulke-options: -t, -p", + 'status': "Cleaning %s", + 'func': 'clean' + }, + 'time': { + "short": "-t", "long": "--time", + "help": "(cleanlogs only): -t, --time Delete logs older than NUM of days", + 'status': "", + 'action': 'store', + 'type': 'int', + 'dest': 'NUM', + 'callback': None, + 'callback_kwargs': None, + 'func': 'clean' + }, + 'pretend': { + "short": "-p", "long": "--pretend", + "help": "(cleanlogs only): -p, --pretend Output logs that would be deleted", + 'status': "", + 'action': 'store_true', + 'dest': 'pretend', + 'callback': None, + 'callback_kwargs': None, + 'func': 'clean' + } + } + } + } + } diff --git a/pym/portage/emaint/modules/logs/logs.py b/pym/portage/emaint/modules/logs/logs.py new file mode 100644 index 00000000000..32c8508f70f --- /dev/null +++ b/pym/portage/emaint/modules/logs/logs.py @@ -0,0 +1,114 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.util import shlex_split, varexpand + +## default clean command from make.globals +## PORT_LOGDIR_CLEAN = 'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +7 -delete' + +class CleanLogs(object): + + short_desc = "Clean PORT_LOGDIR logs" + + def name(): + return "logs" + name = staticmethod(name) + + + def can_progressbar(self, func): + return False + + + def check(self, **kwargs): + if kwargs: + options = kwargs.get('options', None) + if options: + options['pretend'] = True + return self.clean(**kwargs) + + + def clean(self, **kwargs): + """Log directory cleaning function + + @param **kwargs: optional dictionary of values used in this function are: + settings: portage settings instance: defaults to portage.settings + "PORT_LOGDIR": directory to clean + "PORT_LOGDIR_CLEAN": command for cleaning the logs. + options: dict: + 'NUM': int: number of days + 'pretend': boolean + 'eerror': defaults to None, optional output module to output errors. + 'einfo': defaults to None, optional output module to output info msgs. + """ + messages = [] + num_of_days = None + if kwargs: + # convuluted, I know, but portage.settings does not exist in + # kwargs.get() when called from _emerge.main.clean_logs() + settings = kwargs.get('settings', None) + if not settings: + settings = portage.settings + options = kwargs.get('options', None) + if options: + num_of_days = options.get('NUM', None) + pretend = options.get('pretend', False) + eerror = options.get('eerror', None) + einfo = options.get('einfo', None) + + clean_cmd = settings.get("PORT_LOGDIR_CLEAN") + if clean_cmd: + clean_cmd = shlex_split(clean_cmd) + if '-mtime' in clean_cmd and num_of_days is not None: + if num_of_days == 0: + i = clean_cmd.index('-mtime') + clean_cmd.remove('-mtime') + clean_cmd.pop(i) + else: + clean_cmd[clean_cmd.index('-mtime') +1] = \ + '+%s' % str(num_of_days) + if pretend: + if "-delete" in clean_cmd: + clean_cmd.remove("-delete") + + if not clean_cmd: + return [] + rval = self._clean_logs(clean_cmd, settings) + messages += self._convert_errors(rval, eerror, einfo) + return messages + + + @staticmethod + def _clean_logs(clean_cmd, settings): + logdir = settings.get("PORT_LOGDIR") + if logdir is None or not os.path.isdir(logdir): + return + + variables = {"PORT_LOGDIR" : logdir} + cmd = [varexpand(x, mydict=variables) for x in clean_cmd] + + try: + rval = portage.process.spawn(cmd, env=os.environ) + except portage.exception.CommandNotFound: + rval = 127 + return rval + + + @staticmethod + def _convert_errors(rval, eerror=None, einfo=None): + msg = [] + if rval != os.EX_OK: + msg.append("PORT_LOGDIR_CLEAN command returned %s" + % ("%d" % rval if rval else "None")) + msg.append("See the make.conf(5) man page for " + "PORT_LOGDIR_CLEAN usage instructions.") + if eerror: + for m in msg: + eerror(m) + else: + msg.append("PORT_LOGDIR_CLEAN command succeeded") + if einfo: + for m in msg: + einfo(m) + return msg diff --git a/pym/portage/emaint/modules/move/__init__.py b/pym/portage/emaint/modules/move/__init__.py new file mode 100644 index 00000000000..5399440cec9 --- /dev/null +++ b/pym/portage/emaint/modules/move/__init__.py @@ -0,0 +1,33 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: + 1) "Performing package move updates for installed packages", + 2)"Perform package move updates for binary packages" +""" + + +module_spec = { + 'name': 'move', + 'description': "Provides functions to check for and move packages " +\ + "either installed or binary packages stored on this system", + 'provides':{ + 'module1': { + 'name': "moveinst", + 'class': "MoveInstalled", + 'description': "Perform package move updates for installed packages", + 'options': ['check', 'fix'], + 'functions': ['check', 'fix'], + 'func_desc': { + } + }, + 'module2':{ + 'name': "movebin", + 'class': "MoveBinary", + 'description': "Perform package move updates for binary packages", + 'functions': ['check', 'fix'], + 'func_desc': { + } + } + } + } diff --git a/pym/portage/emaint/modules/move/move.py b/pym/portage/emaint/modules/move/move.py new file mode 100644 index 00000000000..018e6cac1ac --- /dev/null +++ b/pym/portage/emaint/modules/move/move.py @@ -0,0 +1,162 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os + + +class MoveHandler(object): + + def __init__(self, tree, porttree): + self._tree = tree + self._portdb = porttree.dbapi + self._update_keys = ["DEPEND", "RDEPEND", "PDEPEND", "PROVIDE"] + self._master_repo = \ + self._portdb.getRepositoryName(self._portdb.porttree_root) + + def _grab_global_updates(self): + from portage.update import grab_updates, parse_updates + retupdates = {} + errors = [] + + for repo_name in self._portdb.getRepositories(): + repo = self._portdb.getRepositoryPath(repo_name) + updpath = os.path.join(repo, "profiles", "updates") + if not os.path.isdir(updpath): + continue + + try: + rawupdates = grab_updates(updpath) + except portage.exception.DirectoryNotFound: + rawupdates = [] + upd_commands = [] + for mykey, mystat, mycontent in rawupdates: + commands, errors = parse_updates(mycontent) + upd_commands.extend(commands) + errors.extend(errors) + retupdates[repo_name] = upd_commands + + if self._master_repo in retupdates: + retupdates['DEFAULT'] = retupdates[self._master_repo] + + return retupdates, errors + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + allupdates, errors = self._grab_global_updates() + # Matching packages and moving them is relatively fast, so the + # progress bar is updated in indeterminate mode. + match = self._tree.dbapi.match + aux_get = self._tree.dbapi.aux_get + if onProgress: + onProgress(0, 0) + for repo, updates in allupdates.items(): + if repo == 'DEFAULT': + continue + if not updates: + continue + + def repo_match(repository): + return repository == repo or \ + (repo == self._master_repo and \ + repository not in allupdates) + + for i, update_cmd in enumerate(updates): + if update_cmd[0] == "move": + origcp, newcp = update_cmd[1:] + for cpv in match(origcp): + if repo_match(aux_get(cpv, ["repository"])[0]): + errors.append("'%s' moved to '%s'" % (cpv, newcp)) + elif update_cmd[0] == "slotmove": + pkg, origslot, newslot = update_cmd[1:] + for cpv in match(pkg): + slot, prepo = aux_get(cpv, ["SLOT", "repository"]) + if slot == origslot and repo_match(prepo): + errors.append("'%s' slot moved from '%s' to '%s'" % \ + (cpv, origslot, newslot)) + if onProgress: + onProgress(0, 0) + + # Searching for updates in all the metadata is relatively slow, so this + # is where the progress bar comes out of indeterminate mode. + cpv_all = self._tree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + meta_keys = self._update_keys + ['repository', 'EAPI'] + if onProgress: + onProgress(maxval, 0) + for i, cpv in enumerate(cpv_all): + metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) + eapi = metadata.pop('EAPI') + repository = metadata.pop('repository') + try: + updates = allupdates[repository] + except KeyError: + try: + updates = allupdates['DEFAULT'] + except KeyError: + continue + if not updates: + continue + metadata_updates = \ + portage.update_dbentries(updates, metadata, eapi=eapi) + if metadata_updates: + errors.append("'%s' has outdated metadata" % cpv) + if onProgress: + onProgress(maxval, i+1) + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + allupdates, errors = self._grab_global_updates() + # Matching packages and moving them is relatively fast, so the + # progress bar is updated in indeterminate mode. + move = self._tree.dbapi.move_ent + slotmove = self._tree.dbapi.move_slot_ent + if onProgress: + onProgress(0, 0) + for repo, updates in allupdates.items(): + if repo == 'DEFAULT': + continue + if not updates: + continue + + def repo_match(repository): + return repository == repo or \ + (repo == self._master_repo and \ + repository not in allupdates) + + for i, update_cmd in enumerate(updates): + if update_cmd[0] == "move": + move(update_cmd, repo_match=repo_match) + elif update_cmd[0] == "slotmove": + slotmove(update_cmd, repo_match=repo_match) + if onProgress: + onProgress(0, 0) + + # Searching for updates in all the metadata is relatively slow, so this + # is where the progress bar comes out of indeterminate mode. + self._tree.dbapi.update_ents(allupdates, onProgress=onProgress) + return errors + +class MoveInstalled(MoveHandler): + + short_desc = "Perform package move updates for installed packages" + + def name(): + return "moveinst" + name = staticmethod(name) + def __init__(self): + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"]) + +class MoveBinary(MoveHandler): + + short_desc = "Perform package move updates for binary packages" + + def name(): + return "movebin" + name = staticmethod(name) + def __init__(self): + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree']) diff --git a/pym/portage/emaint/modules/resume/__init__.py b/pym/portage/emaint/modules/resume/__init__.py new file mode 100644 index 00000000000..60cffe9db6a --- /dev/null +++ b/pym/portage/emaint/modules/resume/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Cleaning the "emerge --resume" lists +""" + + +module_spec = { + 'name': 'resume', + 'description': "Provides functions to scan, check and fix problems " +\ + "in the resume and/or resume_backup files", + 'provides':{ + 'module1': { + 'name': "cleanresume", + 'class': "CleanResume", + 'description': "Discard emerge --resume merge lists", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/resume/resume.py b/pym/portage/emaint/modules/resume/resume.py new file mode 100644 index 00000000000..1bada5288e5 --- /dev/null +++ b/pym/portage/emaint/modules/resume/resume.py @@ -0,0 +1,58 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage + + +class CleanResume(object): + + short_desc = "Discard emerge --resume merge lists" + + def name(): + return "cleanresume" + name = staticmethod(name) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + messages = [] + mtimedb = portage.mtimedb + resume_keys = ("resume", "resume_backup") + maxval = len(resume_keys) + if onProgress: + onProgress(maxval, 0) + for i, k in enumerate(resume_keys): + try: + d = mtimedb.get(k) + if d is None: + continue + if not isinstance(d, dict): + messages.append("unrecognized resume list: '%s'" % k) + continue + mergelist = d.get("mergelist") + if mergelist is None or not hasattr(mergelist, "__len__"): + messages.append("unrecognized resume list: '%s'" % k) + continue + messages.append("resume list '%s' contains %d packages" % \ + (k, len(mergelist))) + finally: + if onProgress: + onProgress(maxval, i+1) + return messages + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + delete_count = 0 + mtimedb = portage.mtimedb + resume_keys = ("resume", "resume_backup") + maxval = len(resume_keys) + if onProgress: + onProgress(maxval, 0) + for i, k in enumerate(resume_keys): + try: + if mtimedb.pop(k, None) is not None: + delete_count += 1 + finally: + if onProgress: + onProgress(maxval, i+1) + if delete_count: + mtimedb.commit() diff --git a/pym/portage/emaint/modules/world/__init__.py b/pym/portage/emaint/modules/world/__init__.py new file mode 100644 index 00000000000..103b5c5ba0b --- /dev/null +++ b/pym/portage/emaint/modules/world/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""'This emaint module provides checks and maintenance for: +Fixing problems with the "world" file. +""" + + +module_spec = { + 'name': 'world', + 'description': "Provides functions to scan, " + + "check and fix problems in the world file", + 'provides':{ + 'module1':{ + 'name': "world", + 'class': "WorldHandler", + 'description': "Fix problems in the world file", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/pym/portage/emaint/modules/world/world.py b/pym/portage/emaint/modules/world/world.py new file mode 100644 index 00000000000..2c9dbffeb3b --- /dev/null +++ b/pym/portage/emaint/modules/world/world.py @@ -0,0 +1,89 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os + + +class WorldHandler(object): + + short_desc = "Fix problems in the world file" + + def name(): + return "world" + name = staticmethod(name) + + def __init__(self): + self.invalid = [] + self.not_installed = [] + self.okay = [] + from portage._sets import load_default_config + setconfig = load_default_config(portage.settings, + portage.db[portage.settings['EROOT']]) + self._sets = setconfig.getSets() + + def _check_world(self, onProgress): + eroot = portage.settings['EROOT'] + self.world_file = os.path.join(eroot, portage.const.WORLD_FILE) + self.found = os.access(self.world_file, os.R_OK) + vardb = portage.db[eroot]["vartree"].dbapi + + from portage._sets import SETPREFIX + sets = self._sets + world_atoms = list(sets["selected"]) + maxval = len(world_atoms) + if onProgress: + onProgress(maxval, 0) + for i, atom in enumerate(world_atoms): + if not isinstance(atom, portage.dep.Atom): + if atom.startswith(SETPREFIX): + s = atom[len(SETPREFIX):] + if s in sets: + self.okay.append(atom) + else: + self.not_installed.append(atom) + else: + self.invalid.append(atom) + if onProgress: + onProgress(maxval, i+1) + continue + okay = True + if not vardb.match(atom): + self.not_installed.append(atom) + okay = False + if okay: + self.okay.append(atom) + if onProgress: + onProgress(maxval, i+1) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + self._check_world(onProgress) + errors = [] + if self.found: + errors += ["'%s' is not a valid atom" % x for x in self.invalid] + errors += ["'%s' is not installed" % x for x in self.not_installed] + else: + errors.append(self.world_file + " could not be opened for reading") + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + world_set = self._sets["selected"] + world_set.lock() + try: + world_set.load() # maybe it's changed on disk + before = set(world_set) + self._check_world(onProgress) + after = set(self.okay) + errors = [] + if before != after: + try: + world_set.replace(self.okay) + except portage.exception.PortageException: + errors.append("%s could not be opened for writing" % \ + self.world_file) + return errors + finally: + world_set.unlock() + diff --git a/pym/portage/emaint/progress.py b/pym/portage/emaint/progress.py new file mode 100644 index 00000000000..e43c2afbd7d --- /dev/null +++ b/pym/portage/emaint/progress.py @@ -0,0 +1,61 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import time +import signal + +import portage + + +class ProgressHandler(object): + def __init__(self): + self.reset() + + def reset(self): + self.curval = 0 + self.maxval = 0 + self.last_update = 0 + self.min_display_latency = 0.2 + + def onProgress(self, maxval, curval): + self.maxval = maxval + self.curval = curval + cur_time = time.time() + if cur_time - self.last_update >= self.min_display_latency: + self.last_update = cur_time + self.display() + + def display(self): + raise NotImplementedError(self) + + +class ProgressBar(ProgressHandler): + """Class to set up and return a Progress Bar""" + + def __init__(self, isatty, **kwargs): + self.isatty = isatty + self.kwargs = kwargs + ProgressHandler.__init__(self) + self.progressBar = None + + def start(self): + if self.isatty: + self.progressBar = portage.output.TermProgressBar(**self.kwargs) + signal.signal(signal.SIGWINCH, self.sigwinch_handler) + else: + self.onProgress = None + return self.onProgress + + def set_label(self, _label): + self.kwargs['label'] = _label + + def display(self): + self.progressBar.set(self.curval, self.maxval) + + def sigwinch_handler(self, signum, frame): + lines, self.progressBar.term_columns = \ + portage.output.get_term_size() + + def stop(self): + signal.signal(signal.SIGWINCH, signal.SIG_DFL) + diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py index 579a46f9b7e..212f78889ab 100644 --- a/pym/portage/getbinpkg.py +++ b/pym/portage/getbinpkg.py @@ -1,5 +1,5 @@ # getbinpkg.py -- Portage binary-package helper functions -# Copyright 2003-2011 Gentoo Foundation +# Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.output import colorize @@ -10,6 +10,7 @@ from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode +from portage.package.ebuild.fetch import _hide_url_passwd from _emerge.Package import _all_metadata_keys import sys @@ -158,11 +159,16 @@ def create_conn(baseurl,conn=None): http_headers = {} http_params = {} if username and password: + try: + encodebytes = base64.encodebytes + except AttributeError: + # Python 2 + encodebytes = base64.encodestring http_headers = { - "Authorization": "Basic %s" % - base64.encodestring("%s:%s" % (username, password)).replace( - "\012", - "" + b"Authorization": "Basic %s" % \ + encodebytes(_unicode_encode("%s:%s" % (username, password))).replace( + b"\012", + b"" ), } @@ -573,7 +579,8 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= try: filelist = dir_get_list(baseurl, conn) except portage.exception.PortageException as e: - sys.stderr.write(_("!!! Error connecting to '%s'.\n") % baseurl) + sys.stderr.write(_("!!! Error connecting to '%s'.\n") % + _hide_url_passwd(baseurl)) sys.stderr.write("!!! %s\n" % str(e)) del e return metadata[baseurl]["data"] diff --git a/pym/portage/glsa.py b/pym/portage/glsa.py index 2df7ec3a7dd..18576957467 100644 --- a/pym/portage/glsa.py +++ b/pym/portage/glsa.py @@ -1,4 +1,4 @@ -# Copyright 2003-2011 Gentoo Foundation +# Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import absolute_import @@ -17,7 +17,7 @@ from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode -from portage.versions import pkgsplit, catpkgsplit, pkgcmp, best +from portage.versions import pkgsplit, vercmp, best from portage.util import grabfile from portage.const import CACHE_PATH from portage.localization import _ @@ -372,17 +372,14 @@ def getMinUpgrade(vulnerableList, unaffectedList, portdbapi, vardbapi, minimize= for u in unaffectedList: mylist = match(u, portdbapi, match_type="match-all") for c in mylist: - c_pv = catpkgsplit(c) - i_pv = catpkgsplit(best(v_installed)) - if pkgcmp(c_pv[1:], i_pv[1:]) > 0 \ + i = best(v_installed) + if vercmp(c.version, i.version) > 0 \ and (rValue == None \ or not match("="+rValue, portdbapi) \ - or (minimize ^ (pkgcmp(c_pv[1:], catpkgsplit(rValue)[1:]) > 0)) \ + or (minimize ^ (vercmp(c.version, rValue.version) > 0)) \ and match("="+c, portdbapi)) \ and portdbapi.aux_get(c, ["SLOT"]) == vardbapi.aux_get(best(v_installed), ["SLOT"]): - rValue = c_pv[0]+"/"+c_pv[1]+"-"+c_pv[2] - if c_pv[3] != "r0": # we don't like -r0 for display - rValue += "-"+c_pv[3] + rValue = c return rValue def format_date(datestr): diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py index 90324eebe15..a04b71780f4 100644 --- a/pym/portage/manifest.py +++ b/pym/portage/manifest.py @@ -8,7 +8,8 @@ import warnings import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all', + 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \ + 'verify_all,_filter_unaccelarated_hashes', 'portage.util:write_atomic', ) @@ -506,9 +507,9 @@ class Manifest(object): self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles) def checkFileHashes(self, ftype, fname, ignoreMissing=False): - myhashes = self.fhashdict[ftype][fname] try: - ok,reason = verify_all(self._getAbsname(ftype, fname), self.fhashdict[ftype][fname]) + ok, reason = verify_all(self._getAbsname(ftype, fname), + _filter_unaccelarated_hashes(self.fhashdict[ftype][fname])) if not ok: raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason))) return ok, reason diff --git a/pym/portage/output.py b/pym/portage/output.py index 98bec814317..e44375ee384 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -7,6 +7,7 @@ import errno import io import formatter import re +import subprocess import sys import portage @@ -425,28 +426,41 @@ class StyleWriter(formatter.DumbWriter): if self.style_listener: self.style_listener(styles) -def get_term_size(): +def get_term_size(fd=None): """ Get the number of lines and columns of the tty that is connected to - stdout. Returns a tuple of (lines, columns) or (0, 0) if an error + fd. Returns a tuple of (lines, columns) or (0, 0) if an error occurs. The curses module is used if available, otherwise the output of `stty size` is parsed. The lines and columns values are guaranteed to be greater than or equal to zero, since a negative COLUMNS variable is known to prevent some commands from working (see bug #394091). """ - if not sys.stdout.isatty(): + if fd is None: + fd = sys.stdout + if not hasattr(fd, 'isatty') or not fd.isatty(): return (0, 0) try: import curses try: - curses.setupterm() + curses.setupterm(term=os.environ.get("TERM", "unknown"), + fd=fd.fileno()) return curses.tigetnum('lines'), curses.tigetnum('cols') except curses.error: pass except ImportError: pass - st, out = portage.subprocess_getstatusoutput('stty size') - if st == os.EX_OK: + + try: + proc = subprocess.Popen(["stty", "size"], + stdout=subprocess.PIPE, stderr=fd) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + # stty command not found + return (0, 0) + + out = _unicode_decode(proc.communicate()[0]) + if proc.wait() == os.EX_OK: out = out.split() if len(out) == 2: try: @@ -631,11 +645,14 @@ class EOutput(object): class ProgressBar(object): """The interface is copied from the ProgressBar class from the EasyDialogs module (which is Mac only).""" - def __init__(self, title=None, maxval=0, label=None): - self._title = title + def __init__(self, title=None, maxval=0, label=None, max_desc_length=25): + self._title = title or "" self._maxval = maxval - self._label = maxval + self._label = label or "" self._curval = 0 + self._desc = "" + self._desc_max_length = max_desc_length + self._set_desc() @property def curval(self): @@ -659,10 +676,23 @@ class ProgressBar(object): def title(self, newstr): """Sets the text in the title bar of the progress dialog to newstr.""" self._title = newstr + self._set_desc() def label(self, newstr): """Sets the text in the progress box of the progress dialog to newstr.""" self._label = newstr + self._set_desc() + + def _set_desc(self): + self._desc = "%s%s" % ( + "%s: " % self._title if self._title else "", + "%s" % self._label if self._label else "" + ) + if len(self._desc) > self._desc_max_length: # truncate if too long + self._desc = "%s..." % self._desc[:self._desc_max_length - 3] + if len(self._desc): + self._desc = self._desc.ljust(self._desc_max_length) + def set(self, value, maxval=None): """ @@ -691,10 +721,10 @@ class ProgressBar(object): class TermProgressBar(ProgressBar): """A tty progress bar similar to wget's.""" - def __init__(self, **kwargs): + def __init__(self, fd=sys.stdout, **kwargs): ProgressBar.__init__(self, **kwargs) - lines, self.term_columns = get_term_size() - self.file = sys.stdout + lines, self.term_columns = get_term_size(fd) + self.file = fd self._min_columns = 11 self._max_columns = 80 # for indeterminate mode, ranges from 0.0 to 1.0 @@ -717,16 +747,18 @@ class TermProgressBar(ProgressBar): curval = self._curval maxval = self._maxval position = self._position - percentage_str_width = 4 + percentage_str_width = 5 square_brackets_width = 2 if cols < percentage_str_width: return "" - bar_space = cols - percentage_str_width - square_brackets_width + bar_space = cols - percentage_str_width - square_brackets_width - 1 + if self._desc: + bar_space -= self._desc_max_length if maxval == 0: max_bar_width = bar_space-3 - image = " " + _percent = "".ljust(percentage_str_width) if cols < min_columns: - return image + return "" if position <= 0.5: offset = 2 * position else: @@ -742,16 +774,16 @@ class TermProgressBar(ProgressBar): position = 0.5 self._position = position bar_width = int(offset * max_bar_width) - image = image + "[" + (bar_width * " ") + \ - "<=>" + ((max_bar_width - bar_width) * " ") + "]" + image = "%s%s%s" % (self._desc, _percent, + "[" + (bar_width * " ") + \ + "<=>" + ((max_bar_width - bar_width) * " ") + "]") return image else: percentage = int(100 * float(curval) / maxval) - if percentage == 100: - percentage_str_width += 1 - bar_space -= 1 max_bar_width = bar_space - 1 - image = ("%d%% " % percentage).rjust(percentage_str_width) + _percent = ("%d%% " % percentage).rjust(percentage_str_width) + image = "%s%s" % (self._desc, _percent) + if cols < min_columns: return image offset = float(curval) / maxval diff --git a/pym/portage/package/ebuild/_config/KeywordsManager.py b/pym/portage/package/ebuild/_config/KeywordsManager.py index 2f9f7b30557..0c613ce04f9 100644 --- a/pym/portage/package/ebuild/_config/KeywordsManager.py +++ b/pym/portage/package/ebuild/_config/KeywordsManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -11,7 +11,7 @@ from portage.dep import ExtendedAtomDict, _repo_separator, _slot_separator from portage.localization import _ from portage.package.ebuild._config.helper import ordered_by_atom_specificity from portage.util import grabdict_package, stack_lists, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str class KeywordsManager(object): """Manager class to handle keywords processing and validation""" @@ -77,10 +77,11 @@ class KeywordsManager(object): def getKeywords(self, cpv, slot, keywords, repo): - cp = cpv_getkey(cpv) - pkg = "".join((cpv, _slot_separator, slot)) - if repo and repo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, repo)) + if not hasattr(cpv, 'slot'): + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + cp = pkg.cp keywords = [[x for x in keywords.split() if x != "-*"]] for pkeywords_dict in self._pkeywords_list: cpdict = pkeywords_dict.get(cp) @@ -260,18 +261,19 @@ class KeywordsManager(object): """ pgroups = global_accept_keywords.split() + if not hasattr(cpv, 'slot'): + cpv = _pkg_str(cpv, slot=slot, repo=repo) cp = cpv_getkey(cpv) unmaskgroups = [] if self._p_accept_keywords: - cpv_slot = "%s:%s" % (cpv, slot) accept_keywords_defaults = tuple('~' + keyword for keyword in \ pgroups if keyword[:1] not in "~-") for d in self._p_accept_keywords: cpdict = d.get(cp) if cpdict: pkg_accept_keywords = \ - ordered_by_atom_specificity(cpdict, cpv_slot) + ordered_by_atom_specificity(cpdict, cpv) if pkg_accept_keywords: for x in pkg_accept_keywords: if not x: @@ -280,9 +282,8 @@ class KeywordsManager(object): pkgdict = self.pkeywordsdict.get(cp) if pkgdict: - cpv_slot = "%s:%s" % (cpv, slot) pkg_accept_keywords = \ - ordered_by_atom_specificity(pkgdict, cpv_slot, repo=repo) + ordered_by_atom_specificity(pkgdict, cpv) if pkg_accept_keywords: for x in pkg_accept_keywords: unmaskgroups.extend(x) diff --git a/pym/portage/package/ebuild/_config/LicenseManager.py b/pym/portage/package/ebuild/_config/LicenseManager.py index effd55be3f2..f76e7e2feec 100644 --- a/pym/portage/package/ebuild/_config/LicenseManager.py +++ b/pym/portage/package/ebuild/_config/LicenseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 201-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -10,7 +10,7 @@ from portage.dep import ExtendedAtomDict, use_reduce from portage.exception import InvalidDependString from portage.localization import _ from portage.util import grabdict, grabdict_package, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity @@ -119,8 +119,9 @@ class LicenseManager(object): cp = cpv_getkey(cpv) cpdict = self._plicensedict.get(cp) if cpdict: - cpv_slot = "%s:%s" % (cpv, slot) - plicence_list = ordered_by_atom_specificity(cpdict, cpv_slot, repo) + if not hasattr(cpv, slot): + cpv = _pkg_str(cpv, slot=slot, repo=repo) + plicence_list = ordered_by_atom_specificity(cpdict, cpv) if plicence_list: accept_license = list(self._accept_license) for x in plicence_list: diff --git a/pym/portage/package/ebuild/_config/LocationsManager.py b/pym/portage/package/ebuild/_config/LocationsManager.py index 8ad41f74776..f7a1177e726 100644 --- a/pym/portage/package/ebuild/_config/LocationsManager.py +++ b/pym/portage/package/ebuild/_config/LocationsManager.py @@ -17,7 +17,8 @@ from portage.exception import DirectoryNotFound, ParseError from portage.localization import _ from portage.util import ensure_dirs, grabfile, \ normalize_path, shlex_split, writemsg -from portage.repository.config import parse_layout_conf +from portage.repository.config import parse_layout_conf, \ + _portage1_profiles_allow_directories _PORTAGE1_DIRECTORIES = frozenset([ @@ -28,6 +29,9 @@ _PORTAGE1_DIRECTORIES = frozenset([ _profile_node = collections.namedtuple('_profile_node', 'location portage1_directories') +_allow_parent_colon = frozenset( + ["portage-2"]) + class LocationsManager(object): def __init__(self, config_root=None, eprefix=None, config_profile_path=None, local_config=True, \ @@ -52,11 +56,20 @@ class LocationsManager(object): self.abs_user_config = os.path.join(self.config_root, USER_CONFIG_PATH) self.config_profile_path = config_profile_path - def load_profiles(self, known_repository_paths): - known_repos = set(os.path.realpath(x) for x in known_repository_paths) - # force a trailing '/' for ease of doing startswith checks - known_repos = tuple((x + '/', parse_layout_conf(x)[0]) - for x in known_repos) + def load_profiles(self, repositories, known_repository_paths): + known_repository_paths = set(os.path.realpath(x) + for x in known_repository_paths) + + known_repos = [] + for x in known_repository_paths: + try: + layout_data = {"profile-formats": + repositories.get_repo_for_location(x).profile_formats} + except KeyError: + layout_data = parse_layout_conf(x)[0] + # force a trailing '/' for ease of doing startswith checks + known_repos.append((x + '/', layout_data)) + known_repos = tuple(known_repos) if self.config_profile_path is None: self.config_profile_path = \ @@ -84,12 +97,13 @@ class LocationsManager(object): if self.profile_path: try: self._addProfile(os.path.realpath(self.profile_path), - known_repos) + repositories, known_repos) except ParseError as e: writemsg(_("!!! Unable to parse profile: '%s'\n") % \ self.profile_path, noiselevel=-1) writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) self.profiles = [] + self.profiles_complex = [] if self._user_config and self.profiles: custom_prof = os.path.join( @@ -110,9 +124,10 @@ class LocationsManager(object): noiselevel=-1) raise DirectoryNotFound(var) - def _addProfile(self, currentPath, known_repos): + def _addProfile(self, currentPath, repositories, known_repos): current_abs_path = os.path.abspath(currentPath) allow_directories = True + allow_parent_colon = True repo_loc = None compat_mode = False intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] @@ -120,9 +135,11 @@ class LocationsManager(object): # protect against nested repositories. Insane configuration, but the longest # path will be the correct one. repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) - allow_directories = any(x.startswith("portage-1") + allow_directories = any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats']) compat_mode = layout_data['profile-formats'] == ('portage-1-compat',) + allow_parent_colon = any(x in _allow_parent_colon + for x in layout_data['profile-formats']) if compat_mode: offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) @@ -164,6 +181,12 @@ class LocationsManager(object): _("Empty parent file: '%s'") % parentsFile) for parentPath in parents: abs_parent = parentPath[:1] == os.sep + if not abs_parent and allow_parent_colon: + parentPath = self._expand_parent_colon(parentsFile, + parentPath, repo_loc, repositories) + + # NOTE: This os.path.join() call is intended to ignore + # currentPath if parentPath is already absolute. parentPath = normalize_path(os.path.join( currentPath, parentPath)) @@ -174,7 +197,7 @@ class LocationsManager(object): parentPath = os.path.realpath(parentPath) if os.path.exists(parentPath): - self._addProfile(parentPath, known_repos) + self._addProfile(parentPath, repositories, known_repos) else: raise ParseError( _("Parent '%s' not found: '%s'") % \ @@ -184,6 +207,34 @@ class LocationsManager(object): self.profiles_complex.append( _profile_node(currentPath, allow_directories)) + def _expand_parent_colon(self, parentsFile, parentPath, + repo_loc, repositories): + colon = parentPath.find(":") + if colon == -1: + return parentPath + + if colon == 0: + if repo_loc is None: + raise ParseError( + _("Parent '%s' not found: '%s'") % \ + (parentPath, parentsFile)) + else: + parentPath = normalize_path(os.path.join( + repo_loc, 'profiles', parentPath[colon+1:])) + else: + p_repo_name = parentPath[:colon] + try: + p_repo_loc = repositories.get_location_for_name(p_repo_name) + except KeyError: + raise ParseError( + _("Parent '%s' not found: '%s'") % \ + (parentPath, parentsFile)) + else: + parentPath = normalize_path(os.path.join( + p_repo_loc, 'profiles', parentPath[colon+1:])) + + return parentPath + def set_root_override(self, root_overwrite=None): # Allow ROOT setting to come from make.conf if it's not overridden # by the constructor argument (from the calling environment). diff --git a/pym/portage/package/ebuild/_config/UseManager.py b/pym/portage/package/ebuild/_config/UseManager.py index 0506af01994..e1ec7f4a01f 100644 --- a/pym/portage/package/ebuild/_config/UseManager.py +++ b/pym/portage/package/ebuild/_config/UseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -7,10 +7,10 @@ __all__ = ( from _emerge.Package import Package from portage import os -from portage.dep import ExtendedAtomDict, remove_slot, _get_useflag_re +from portage.dep import dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re from portage.localization import _ from portage.util import grabfile, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity @@ -148,9 +148,13 @@ class UseManager(object): return frozenset(stack_lists( self._usemask_list, incremental=True)) + slot = None cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp usemask = [] if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] diff --git a/pym/portage/package/ebuild/_config/helper.py b/pym/portage/package/ebuild/_config/helper.py index 4f467818756..ee0c090a043 100644 --- a/pym/portage/package/ebuild/_config/helper.py +++ b/pym/portage/package/ebuild/_config/helper.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -24,7 +24,7 @@ def ordered_by_atom_specificity(cpdict, pkg, repo=None): order to achieve desired results (and thus corrupting the ChangeLog like ordering of the file). """ - if repo and repo != Package.UNKNOWN_REPO: + if not hasattr(pkg, 'repo') and repo and repo != Package.UNKNOWN_REPO: pkg = pkg + _repo_separator + repo results = [] diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py b/pym/portage/package/ebuild/_config/special_env_vars.py index 132f8ebd863..6ed6d05423e 100644 --- a/pym/portage/package/ebuild/_config/special_env_vars.py +++ b/pym/portage/package/ebuild/_config/special_env_vars.py @@ -50,8 +50,12 @@ environ_whitelist += [ "PORTAGE_BINPKG_TMPFILE", "PORTAGE_BIN_PATH", "PORTAGE_BUILDDIR", "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", - "PORTAGE_COLORMAP", "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", + "PORTAGE_COLORMAP", "PORTAGE_COMPRESS", + "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", "PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES", + "PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES", "PORTAGE_EBUILD_EXIT_FILE", "PORTAGE_FEATURES", "PORTAGE_GID", "PORTAGE_GRPNAME", "PORTAGE_INST_GID", "PORTAGE_INST_UID", @@ -134,8 +138,9 @@ environ_filter += [ # portage config variables and variables set directly by portage environ_filter += [ "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", "AUTOCLEAN", - "CLEAN_DELAY", "COLLISION_IGNORE", "CONFIG_PROTECT", - "CONFIG_PROTECT_MASK", "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", + "CLEAN_DELAY", "COLLISION_IGNORE", + "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", + "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", "EMERGE_LOG_DIR", "EMERGE_WARNING_DELAY", "FETCHCOMMAND", "FETCHCOMMAND_FTP", @@ -162,7 +167,7 @@ environ_filter += [ "RESUMECOMMAND", "RESUMECOMMAND_FTP", "RESUMECOMMAND_HTTP", "RESUMECOMMAND_HTTPS", "RESUMECOMMAND_RSYNC", "RESUMECOMMAND_SFTP", - "SYNC", "USE_EXPAND_HIDDEN", "USE_ORDER", + "SYNC", "UNINSTALL_IGNORE", "USE_EXPAND_HIDDEN", "USE_ORDER", ] environ_filter = frozenset(environ_filter) diff --git a/pym/portage/package/ebuild/_eapi_invalid.py b/pym/portage/package/ebuild/_eapi_invalid.py new file mode 100644 index 00000000000..d23677d236c --- /dev/null +++ b/pym/portage/package/ebuild/_eapi_invalid.py @@ -0,0 +1,54 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +import portage +from portage.dep import _repo_separator +from portage.elog import elog_process +from portage.elog.messages import eerror + +def eapi_invalid(self, cpv, repo_name, settings, + eapi_var, eapi_parsed, eapi_lineno): + + msg = [] + msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" + " conform with PMS section 7.3.1 (see bug #402167):") % + (cpv, _repo_separator, repo_name), 70)) + + if not eapi_parsed: + # None means the assignment was not found, while an + # empty string indicates an (invalid) empty assingment. + msg.append( + "\tvalid EAPI assignment must" + " occur on or before line: %s" % + eapi_lineno) + else: + msg.append(("\tbash returned EAPI '%s' which does not match " + "assignment on line: %s") % + (eapi_var, eapi_lineno)) + + if 'parse-eapi-ebuild-head' in settings.features: + msg.extend(textwrap.wrap(("NOTE: This error will soon" + " become unconditionally fatal in a future version of Portage," + " but at this time, it can by made non-fatal by setting" + " FEATURES=-parse-eapi-ebuild-head in" + " make.conf."), 70)) + else: + msg.extend(textwrap.wrap(("NOTE: This error will soon" + " become unconditionally fatal in a future version of Portage." + " At the earliest opportunity, please enable" + " FEATURES=parse-eapi-ebuild-head in make.conf in order to" + " make this error fatal."), 70)) + + if portage.data.secpass >= 2: + # TODO: improve elog permission error handling (bug #416231) + for line in msg: + eerror(line, phase="other", key=cpv) + elog_process(cpv, settings, + phasefilter=("other",)) + + else: + out = portage.output.EOutput() + for line in msg: + out.eerror(line) diff --git a/pym/portage/package/ebuild/_ipc/QueryCommand.py b/pym/portage/package/ebuild/_ipc/QueryCommand.py index 7bbb0e83bdc..d200fe80d12 100644 --- a/pym/portage/package/ebuild/_ipc/QueryCommand.py +++ b/pym/portage/package/ebuild/_ipc/QueryCommand.py @@ -20,6 +20,12 @@ class QueryCommand(IpcCommand): _db = None + @classmethod + def get_db(cls): + if cls._db is not None: + return cls._db + return portage.db + def __init__(self, settings, phase): IpcCommand.__init__(self) self.settings = settings @@ -52,9 +58,7 @@ class QueryCommand(IpcCommand): use = frozenset(use.split()) atom = atom.evaluate_conditionals(use) - db = self._db - if db is None: - db = portage.db + db = self.get_db() warnings_str = '' if warnings: diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index 2cd21f3c918..2fa799f7e3d 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -43,7 +43,7 @@ from portage.util import ensure_dirs, getconfig, grabdict, \ grabdict_package, grabfile, grabfile_package, LazyItemsDict, \ normalize_path, shlex_split, stack_dictlist, stack_dicts, stack_lists, \ writemsg, writemsg_level -from portage.versions import catpkgsplit, catsplit, cpv_getkey +from portage.versions import catpkgsplit, catsplit, cpv_getkey, _pkg_str from portage.package.ebuild._config import special_env_vars from portage.package.ebuild._config.env_var_validation import validate_cmd_var @@ -181,6 +181,7 @@ class config(object): # can practically render the api unusable for api consumers. tolerant = hasattr(portage, '_initializing_globals') self._tolerant = tolerant + self._unmatched_removal = _unmatched_removal self.locked = 0 self.mycpv = None @@ -205,6 +206,7 @@ class config(object): # For immutable attributes, use shallow copy for # speed and memory conservation. self._tolerant = clone._tolerant + self._unmatched_removal = clone._unmatched_removal self.categories = clone.categories self.depcachedir = clone.depcachedir self.incrementals = clone.incrementals @@ -227,9 +229,12 @@ class config(object): self._setcpv_args_hash = clone._setcpv_args_hash # immutable attributes (internal policy ensures lack of mutation) - self._keywords_manager = clone._keywords_manager + self._locations_manager = clone._locations_manager self._use_manager = clone._use_manager - self._mask_manager = clone._mask_manager + # force instantiation of lazy immutable objects when cloning, so + # that they're not instantiated more than once + self._keywords_manager_obj = clone._keywords_manager + self._mask_manager_obj = clone._mask_manager # shared mutable attributes self._unknown_features = clone._unknown_features @@ -266,7 +271,9 @@ class config(object): #all LicenseManager instances. self._license_manager = clone._license_manager - self._virtuals_manager = copy.deepcopy(clone._virtuals_manager) + # force instantiation of lazy objects when cloning, so + # that they're not instantiated more than once + self._virtuals_manager_obj = copy.deepcopy(clone._virtuals_manager) self._accept_properties = copy.deepcopy(clone._accept_properties) self._ppropertiesdict = copy.deepcopy(clone._ppropertiesdict) @@ -274,9 +281,15 @@ class config(object): self._expand_map = copy.deepcopy(clone._expand_map) else: + # lazily instantiated objects + self._keywords_manager_obj = None + self._mask_manager_obj = None + self._virtuals_manager_obj = None + locations_manager = LocationsManager(config_root=config_root, config_profile_path=config_profile_path, eprefix=eprefix, local_config=local_config, target_root=target_root) + self._locations_manager = locations_manager eprefix = locations_manager.eprefix config_root = locations_manager.config_root @@ -313,17 +326,18 @@ class config(object): # Notably absent is "env", since we want to avoid any # interaction with the calling environment that might # lead to unexpected results. - expand_map = {} + + env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), + tolerant=tolerant, expand=False) or {} + expand_map = env_d.copy() self._expand_map = expand_map # Allow make.globals to set default paths relative to ${EPREFIX}. expand_map["EPREFIX"] = eprefix - env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), - expand=False) - make_globals = getconfig(os.path.join( - self.global_config_path, 'make.globals'), expand=expand_map) + self.global_config_path, 'make.globals'), + tolerant=tolerant, expand=expand_map) if make_globals is None: make_globals = {} @@ -404,13 +418,30 @@ class config(object): self.make_defaults_use = [] + #Loading Repositories + self["PORTAGE_CONFIGROOT"] = config_root + self["ROOT"] = target_root + self["EPREFIX"] = eprefix + self["EROOT"] = eroot known_repos = [] + portdir = "" + portdir_overlay = "" for confs in [make_globals, make_conf, self.configdict["env"]]: - known_repos.extend(confs.get("PORTDIR", '').split()) - known_repos.extend(confs.get("PORTDIR_OVERLAY", '').split()) + v = confs.get("PORTDIR") + if v is not None: + portdir = v + known_repos.append(v) + v = confs.get("PORTDIR_OVERLAY") + if v is not None: + portdir_overlay = v + known_repos.extend(shlex_split(v)) known_repos = frozenset(known_repos) + self["PORTDIR"] = portdir + self["PORTDIR_OVERLAY"] = portdir_overlay + self.lookuplist = [self.configdict["env"]] + self.repositories = load_repository_config(self) - locations_manager.load_profiles(known_repos) + locations_manager.load_profiles(self.repositories, known_repos) profiles_complex = locations_manager.profiles_complex self.profiles = locations_manager.profiles @@ -433,7 +464,8 @@ class config(object): mygcfg = {} if self.profiles: mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults"), - expand=expand_map) for x in self.profiles] + tolerant=tolerant, expand=expand_map) + for x in self.profiles] self._make_defaults = mygcfg_dlists mygcfg = stack_dicts(mygcfg_dlists, incrementals=self.incrementals) @@ -517,14 +549,10 @@ class config(object): self._ppropertiesdict = portage.dep.ExtendedAtomDict(dict) self._penvdict = portage.dep.ExtendedAtomDict(dict) - #Loading Repositories - self.repositories = load_repository_config(self) - #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility main_repo = self.repositories.mainRepo() if main_repo is not None: - main_repo = main_repo.user_location - self["PORTDIR"] = main_repo + self["PORTDIR"] = main_repo.user_location self.backup_changes("PORTDIR") # repoman controls PORTDIR_OVERLAY via the environment, so no @@ -554,17 +582,13 @@ class config(object): self._repo_make_defaults = {} for repo in self.repositories.repos_with_profiles(): d = getconfig(os.path.join(repo.location, "profiles", "make.defaults"), - expand=self.configdict["globals"].copy()) or {} + tolerant=tolerant, expand=self.configdict["globals"].copy()) or {} if d: for k in chain(self._env_blacklist, profile_only_variables, self._global_only_vars): d.pop(k, None) self._repo_make_defaults[repo.name] = d - #Read package.keywords and package.accept_keywords. - self._keywords_manager = KeywordsManager(profiles_complex, abs_user_config, \ - local_config, global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) - #Read all USE related files from profiles and optionally from user config. self._use_manager = UseManager(self.repositories, profiles_complex, abs_user_config, user_config=local_config) #Initialize all USE related variables we track ourselves. @@ -582,13 +606,6 @@ class config(object): self._license_manager.extract_global_changes( \ self.configdict["conf"].get("ACCEPT_LICENSE", "")) - #Read package.mask and package.unmask from profiles and optionally from user config - self._mask_manager = MaskManager(self.repositories, profiles_complex, - abs_user_config, user_config=local_config, - strict_umatched_removal=_unmatched_removal) - - self._virtuals_manager = VirtualsManager(self.profiles) - if local_config: #package.properties propdict = grabdict_package(os.path.join( @@ -776,9 +793,6 @@ class config(object): if bsd_chflags: self.features.add('chflags') - if 'parse-eapi-ebuild-head' in self.features: - portage._validate_cache_for_unsupported_eapis = False - self._iuse_implicit_match = _iuse_implicit_match_cache(self) self._validate_commands() @@ -788,6 +802,11 @@ class config(object): self[k] = self[k].lower() self.backup_changes(k) + if main_repo is not None and not main_repo.sync: + main_repo_sync = self.get("SYNC") + if main_repo_sync: + main_repo.sync = main_repo_sync + # The first constructed config object initializes these modules, # and subsequent calls to the _init() functions have no effect. portage.output._init(config_root=self['PORTAGE_CONFIGROOT']) @@ -867,6 +886,32 @@ class config(object): noiselevel=-1) @property + def _keywords_manager(self): + if self._keywords_manager_obj is None: + self._keywords_manager_obj = KeywordsManager( + self._locations_manager.profiles_complex, + self._locations_manager.abs_user_config, + self.local_config, + global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) + return self._keywords_manager_obj + + @property + def _mask_manager(self): + if self._mask_manager_obj is None: + self._mask_manager_obj = MaskManager(self.repositories, + self._locations_manager.profiles_complex, + self._locations_manager.abs_user_config, + user_config=self.local_config, + strict_umatched_removal=self._unmatched_removal) + return self._mask_manager_obj + + @property + def _virtuals_manager(self): + if self._virtuals_manager_obj is None: + self._virtuals_manager_obj = VirtualsManager(self.profiles) + return self._virtuals_manager_obj + + @property def pkeywordsdict(self): result = self._keywords_manager.pkeywordsdict.copy() for k, v in result.items(): @@ -904,11 +949,26 @@ class config(object): writemsg(_("!!! INVALID ACCEPT_KEYWORDS: %s\n") % str(group), noiselevel=-1) - abs_profile_path = os.path.join(self["PORTAGE_CONFIGROOT"], - PROFILE_PATH) - if (not self.profile_path or \ - not os.path.exists(os.path.join(self.profile_path, "parent"))) and \ - os.path.exists(os.path.join(self["PORTDIR"], "profiles")): + profile_broken = not self.profile_path or \ + not os.path.exists(os.path.join(self.profile_path, "parent")) and \ + os.path.exists(os.path.join(self["PORTDIR"], "profiles")) + + if profile_broken: + abs_profile_path = None + for x in (PROFILE_PATH, 'etc/portage/make.profile'): + x = os.path.join(self["PORTAGE_CONFIGROOT"], x) + try: + os.lstat(x) + except OSError: + pass + else: + abs_profile_path = x + break + + if abs_profile_path is None: + abs_profile_path = os.path.join(self["PORTAGE_CONFIGROOT"], + PROFILE_PATH) + writemsg(_("\n\n!!! %s is not a symlink and will probably prevent most merges.\n") % abs_profile_path, noiselevel=-1) writemsg(_("!!! It should point into a profile within %s/profiles/\n") % self["PORTDIR"]) @@ -1224,7 +1284,7 @@ class config(object): slot = pkg_configdict["SLOT"] iuse = pkg_configdict["IUSE"] if pkg is None: - cpv_slot = "%s:%s" % (self.mycpv, slot) + cpv_slot = _pkg_str(self.mycpv, slot=slot, repo=repository) else: cpv_slot = pkg pkginternaluse = [] @@ -1676,11 +1736,13 @@ class config(object): @return: A list of properties that have not been accepted. """ accept_properties = self._accept_properties + if not hasattr(cpv, 'slot'): + cpv = _pkg_str(cpv, slot=metadata["SLOT"], + repo=metadata.get("repository")) cp = cpv_getkey(cpv) cpdict = self._ppropertiesdict.get(cp) if cpdict: - cpv_slot = "%s:%s" % (cpv, metadata["SLOT"]) - pproperties_list = ordered_by_atom_specificity(cpdict, cpv_slot, repo=metadata.get('repository')) + pproperties_list = ordered_by_atom_specificity(cpdict, cpv) if pproperties_list: accept_properties = list(self._accept_properties) for x in pproperties_list: @@ -1808,7 +1870,8 @@ class config(object): """Reload things like /etc/profile.env that can change during runtime.""" env_d_filename = os.path.join(self["EROOT"], "etc", "profile.env") self.configdict["env.d"].clear() - env_d = getconfig(env_d_filename, expand=False) + env_d = getconfig(env_d_filename, + tolerant=self._tolerant, expand=False) if env_d: # env_d will be None if profile.env doesn't exist. for k in self._env_d_blacklist: diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index 29bef0917a7..09062f9f30b 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -25,13 +25,15 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.digestcheck:digestcheck', 'portage.package.ebuild.digestgen:digestgen', 'portage.package.ebuild.fetch:fetch', + 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', + 'portage.dep._slot_abi:evaluate_slot_abi_equal_deps', 'portage.package.ebuild._spawn_nofetch:spawn_nofetch', 'portage.util.ExtractKernelVersion:ExtractKernelVersion' ) from portage import auxdbkeys, bsd_chflags, \ eapi_is_supported, merge, os, selinux, shutil, \ - unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, \ + unmerge, _encodings, _os_merge, \ _shell_quote, _unicode_decode, _unicode_encode from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_ENV_DIR, \ EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY @@ -43,7 +45,7 @@ from portage.dep import Atom, check_required_use, \ from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \ eapi_exports_replace_vars, eapi_exports_REPOSITORY, \ eapi_has_required_use, eapi_has_src_prepare_and_src_configure, \ - eapi_has_pkg_pretend + eapi_has_pkg_pretend, _get_eapi_attrs from portage.elog import elog_process, _preload_elog_modules from portage.elog.messages import eerror, eqawarn from portage.exception import DigestException, FileNotFound, \ @@ -175,7 +177,7 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, pkg_dir = os.path.dirname(ebuild_path) mytree = os.path.dirname(os.path.dirname(pkg_dir)) mypv = os.path.basename(ebuild_path)[:-7] - mysplit = _pkgsplit(mypv) + mysplit = _pkgsplit(mypv, eapi=mysettings.configdict["pkg"].get("EAPI")) if mysplit is None: raise IncorrectParameter( _("Invalid ebuild path: '%s'") % myebuild) @@ -256,7 +258,6 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings['PORTDIR'] = repo.eclass_db.porttrees[0] mysettings['PORTDIR_OVERLAY'] = ' '.join(repo.eclass_db.porttrees[1:]) mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo.name - mysettings.configdict["pkg"]["REPOSITORY"] = repo.name mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) @@ -331,34 +332,22 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, os.environ["COLUMNS"] = columns mysettings["COLUMNS"] = columns - # All EAPI dependent code comes last, so that essential variables - # like PORTAGE_BUILDDIR are still initialized even in cases when + # EAPI is always known here, even for the "depend" phase, because + # EbuildMetadataPhase gets it from _parse_eapi_ebuild_head(). + eapi = mysettings.configdict['pkg']['EAPI'] + _doebuild_path(mysettings, eapi=eapi) + + # All EAPI dependent code comes last, so that essential variables like + # PATH and PORTAGE_BUILDDIR are still initialized even in cases when # UnsupportedAPIException needs to be raised, which can be useful # when uninstalling a package that has corrupt EAPI metadata. - eapi = None - if mydo == 'depend' and 'EAPI' not in mysettings.configdict['pkg']: - if eapi is None and 'parse-eapi-ebuild-head' in mysettings.features: - with io.open(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], - errors='replace') as f: - eapi = _parse_eapi_ebuild_head(f) - - if eapi is not None: - if not eapi_is_supported(eapi): - _doebuild_path(mysettings) - raise UnsupportedAPIException(mycpv, eapi) - mysettings.configdict['pkg']['EAPI'] = eapi + if not eapi_is_supported(eapi): + raise UnsupportedAPIException(mycpv, eapi) - if mydo != "depend": - # Metadata vars such as EAPI and RESTRICT are - # set by the above config.setcpv() call. - eapi = mysettings["EAPI"] - if not eapi_is_supported(eapi): - # can't do anything with this. - _doebuild_path(mysettings) - raise UnsupportedAPIException(mycpv, eapi) + if eapi_exports_REPOSITORY(eapi) and "PORTAGE_REPO_NAME" in mysettings.configdict["pkg"]: + mysettings.configdict["pkg"]["REPOSITORY"] = mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] + if mydo != "depend": if hasattr(mydbapi, "getFetchMap") and \ ("A" not in mysettings.configdict["pkg"] or \ "AA" not in mysettings.configdict["pkg"]): @@ -383,9 +372,6 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, else: mysettings.configdict["pkg"]["AA"] = " ".join(uri_map) - _doebuild_path(mysettings, eapi=eapi) - - if mydo != "depend": ccache = "ccache" in mysettings.features distcc = "distcc" in mysettings.features if ccache or distcc: @@ -405,25 +391,22 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["PATH"] = os.path.join(os.sep, eprefix_lstrip, "usr", libdir, "ccache", "bin") + ":" + mysettings["PATH"] - if not eapi_exports_KV(eapi): - # Discard KV for EAPIs that don't support it. Cache KV is restored - # from the backupenv whenever config.reset() is called. - mysettings.pop('KV', None) - elif mydo != 'depend' and 'KV' not in mysettings and \ - mydo in ('compile', 'config', 'configure', 'info', - 'install', 'nofetch', 'postinst', 'postrm', 'preinst', - 'prepare', 'prerm', 'setup', 'test', 'unpack'): - mykv, err1 = ExtractKernelVersion( - os.path.join(mysettings['EROOT'], "usr/src/linux")) - if mykv: - # Regular source tree - mysettings["KV"] = mykv - else: - mysettings["KV"] = "" - mysettings.backup_changes("KV") - - if mydo != "depend" and not eapi_exports_REPOSITORY(eapi): - mysettings.pop('REPOSITORY', None) + if not eapi_exports_KV(eapi): + # Discard KV for EAPIs that don't support it. Cached KV is restored + # from the backupenv whenever config.reset() is called. + mysettings.pop('KV', None) + elif 'KV' not in mysettings and \ + mydo in ('compile', 'config', 'configure', 'info', + 'install', 'nofetch', 'postinst', 'postrm', 'preinst', + 'prepare', 'prerm', 'setup', 'test', 'unpack'): + mykv, err1 = ExtractKernelVersion( + os.path.join(mysettings['EROOT'], "usr/src/linux")) + if mykv: + # Regular source tree + mysettings["KV"] = mykv + else: + mysettings["KV"] = "" + mysettings.backup_changes("KV") _doebuild_manifest_cache = None _doebuild_broken_ebuilds = set() @@ -1040,6 +1023,13 @@ def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, if mydo == "package" and bintree is not None: bintree.inject(mysettings.mycpv, filename=mysettings["PORTAGE_BINPKG_TMPFILE"]) + else: + if "PORTAGE_BINPKG_TMPFILE" in mysettings: + try: + os.unlink(mysettings["PORTAGE_BINPKG_TMPFILE"]) + except OSError: + pass + elif mydo=="qmerge": # check to ensure install was run. this *only* pops up when users # forget it and are using ebuild @@ -1635,7 +1625,7 @@ def _check_build_log(mysettings, out=None): if f_real is not None: f_real.close() -def _post_src_install_chost_fix(settings): +def _post_src_install_write_metadata(settings): """ It's possible that the ebuild has changed the CHOST variable, so revert it to the initial @@ -1643,6 +1633,8 @@ def _post_src_install_chost_fix(settings): due to local environment settings like in bug #386829. """ + eapi_attrs = _get_eapi_attrs(settings.configdict['pkg']['EAPI']) + build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info') for k in ('IUSE',): @@ -1658,9 +1650,62 @@ def _post_src_install_chost_fix(settings): if v is not None: write_atomic(os.path.join(build_info_dir, k), v + '\n') + with io.open(_unicode_encode(os.path.join(build_info_dir, + 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write(_unicode_decode("%.0f\n" % (time.time(),))) + + use = frozenset(settings['PORTAGE_USE'].split()) + for k in _vdb_use_conditional_keys: + v = settings.configdict['pkg'].get(k) + filename = os.path.join(build_info_dir, k) + if v is None: + try: + os.unlink(filename) + except OSError: + pass + continue + + if k.endswith('DEPEND'): + if eapi_attrs.slot_abi: + continue + token_class = Atom + else: + token_class = None + + v = use_reduce(v, uselist=use, token_class=token_class) + v = paren_enclose(v) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write(_unicode_decode(v + '\n')) + + if eapi_attrs.slot_abi: + deps = evaluate_slot_abi_equal_deps(settings, use, QueryCommand.get_db()) + for k, v in deps.items(): + filename = os.path.join(build_info_dir, k) + if not v: + try: + os.unlink(filename) + except OSError: + pass + continue + with io.open(_unicode_encode(os.path.join(build_info_dir, + k), encoding=_encodings['fs'], errors='strict'), + mode='w', encoding=_encodings['repo.content'], + errors='strict') as f: + f.write(_unicode_decode(v + '\n')) + _vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) -_vdb_use_conditional_atoms = frozenset(['DEPEND', 'PDEPEND', 'RDEPEND']) def _preinst_bsdflags(mysettings): if bsd_chflags: @@ -1829,44 +1874,6 @@ def _post_src_install_uid_fix(mysettings, out): f.write(_unicode_decode(str(size) + '\n')) f.close() - f = io.open(_unicode_encode(os.path.join(build_info_dir, - 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict') - f.write(_unicode_decode("%.0f\n" % (time.time(),))) - f.close() - - use = frozenset(mysettings['PORTAGE_USE'].split()) - for k in _vdb_use_conditional_keys: - v = mysettings.configdict['pkg'].get(k) - filename = os.path.join(build_info_dir, k) - if v is None: - try: - os.unlink(filename) - except OSError: - pass - continue - - if k.endswith('DEPEND'): - token_class = Atom - else: - token_class = None - - v = use_reduce(v, uselist=use, token_class=token_class) - v = paren_enclose(v) - if not v: - try: - os.unlink(filename) - except OSError: - pass - continue - f = io.open(_unicode_encode(os.path.join(build_info_dir, - k), encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='strict') - f.write(_unicode_decode(v + '\n')) - f.close() - _reapply_bsdflags_to_image(mysettings) def _reapply_bsdflags_to_image(mysettings): diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py index 022d3e89a18..60ed04da28c 100644 --- a/pym/portage/package/ebuild/fetch.py +++ b/pym/portage/package/ebuild/fetch.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -25,7 +25,8 @@ portage.proxy.lazyimport.lazyimport(globals(), from portage import OrderedDict, os, selinux, shutil, _encodings, \ _shell_quote, _unicode_encode -from portage.checksum import hashfunc_map, perform_md5, verify_all +from portage.checksum import (hashfunc_map, perform_md5, verify_all, + _filter_unaccelarated_hashes) from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ GLOBAL_CONFIG_PATH from portage.data import portage_gid, portage_uid, secpass, userpriv_groups @@ -46,6 +47,9 @@ _userpriv_spawn_kwargs = ( ("umask", 0o02), ) +def _hide_url_passwd(url): + return re.sub(r'//(.+):.+@(.+)', r'//\1:*password*@\2', url) + def _spawn_fetch(settings, args, **kwargs): """ Spawn a process with appropriate settings for fetching, including @@ -207,6 +211,7 @@ def _check_distfile(filename, digests, eout, show_errors=1): # Zero-byte distfiles are always invalid. return (False, st) else: + digests = _filter_unaccelarated_hashes(digests) if _check_digests(filename, digests, show_errors=show_errors): eout.ebegin("%s %s ;-)" % (os.path.basename(filename), " ".join(sorted(digests)))) @@ -790,8 +795,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout.eend(0) continue else: - verified_ok, reason = verify_all( - myfile_path, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Previously fetched" " file: '%s'\n") % myfile, noiselevel=-1) @@ -813,7 +818,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, eout = EOutput() eout.quiet = \ mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: digests = list(digests) digests.sort() @@ -949,7 +953,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locfetch=fetchcommand command_var = fetchcommand_var writemsg_stdout(_(">>> Downloading '%s'\n") % \ - re.sub(r'//(.+):.+@(.+)/',r'//\1:*password*@\2/', loc)) + _hide_url_passwd(loc)) variables = { "DISTDIR": mysettings["DISTDIR"], "URI": loc, @@ -1048,7 +1052,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # file NOW, for those users who don't have a stable/continuous # net connection. This way we have a chance to try to download # from another mirror... - verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile]) + digests = _filter_unaccelarated_hashes(mydigests[myfile]) + verified_ok, reason = verify_all(myfile_path, digests) if not verified_ok: writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile, noiselevel=-1) @@ -1082,7 +1087,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, else: eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" - digests = mydigests.get(myfile) if digests: eout.ebegin("%s %s ;-)" % \ (myfile, " ".join(sorted(digests)))) diff --git a/pym/portage/package/ebuild/getmaskingstatus.py b/pym/portage/package/ebuild/getmaskingstatus.py index 66d3db528a8..9bf605db6c0 100644 --- a/pym/portage/package/ebuild/getmaskingstatus.py +++ b/pym/portage/package/ebuild/getmaskingstatus.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['getmaskingstatus'] @@ -9,7 +9,7 @@ import portage from portage import eapi_is_supported, _eapi_is_deprecated from portage.localization import _ from portage.package.ebuild.config import config -from portage.versions import catpkgsplit +from portage.versions import catpkgsplit, _pkg_str if sys.hexversion >= 0x3000000: basestring = str @@ -51,9 +51,6 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): metadata = pkg.metadata installed = pkg.installed - mysplit = catpkgsplit(mycpv) - if not mysplit: - raise ValueError(_("invalid CPV: %s") % mycpv) if metadata is None: db_keys = list(portdb._aux_cache_keys) try: @@ -68,6 +65,13 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): else: metadata["USE"] = "" + if not hasattr(mycpv, 'slot'): + try: + mycpv = _pkg_str(mycpv, slot=metadata['SLOT'], + repo=metadata.get('repository')) + except portage.exception.InvalidData: + raise ValueError(_("invalid CPV: %s") % mycpv) + rValue = [] # package.mask checking @@ -79,8 +83,6 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): mygroups = settings._getKeywords(mycpv, metadata) licenses = metadata["LICENSE"] properties = metadata["PROPERTIES"] - if eapi.startswith("-"): - eapi = eapi[1:] if not eapi_is_supported(eapi): return [_MaskReason("EAPI", "EAPI %s" % eapi)] elif _eapi_is_deprecated(eapi) and not installed: diff --git a/pym/portage/package/ebuild/prepare_build_dirs.py b/pym/portage/package/ebuild/prepare_build_dirs.py index 50b14ec91e0..b8fbdc5cf89 100644 --- a/pym/portage/package/ebuild/prepare_build_dirs.py +++ b/pym/portage/package/ebuild/prepare_build_dirs.py @@ -346,13 +346,31 @@ def _prepare_workdir(mysettings): writemsg(_unicode_decode("!!! %s: %s\n") % (_("Permission Denied"), log_subdir), noiselevel=-1) + tmpdir_log_path = os.path.join( + mysettings["T"], "build.log%s" % compress_log_ext) if not logdir_subdir_ok: # NOTE: When sesandbox is enabled, the local SELinux security policies # may not allow output to be piped out of the sesandbox domain. The # current policy will allow it to work when a pty is available, but # not through a normal pipe. See bug #162404. - mysettings["PORTAGE_LOG_FILE"] = os.path.join( - mysettings["T"], "build.log%s" % compress_log_ext) + mysettings["PORTAGE_LOG_FILE"] = tmpdir_log_path + else: + # Create a symlink from tmpdir_log_path to PORTAGE_LOG_FILE, as + # requested in bug #412865. + make_new_symlink = False + try: + target = os.readlink(tmpdir_log_path) + except OSError: + make_new_symlink = True + else: + if target != mysettings["PORTAGE_LOG_FILE"]: + make_new_symlink = True + if make_new_symlink: + try: + os.unlink(tmpdir_log_path) + except OSError: + pass + os.symlink(mysettings["PORTAGE_LOG_FILE"], tmpdir_log_path) def _ensure_log_subdirs(logdir, subdir): """ diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py index defdb47c86a..9b43f387284 100644 --- a/pym/portage/repository/config.py +++ b/pym/portage/repository/config.py @@ -27,6 +27,12 @@ from portage import _unicode_encode from portage import _encodings from portage import manifest +_valid_profile_formats = frozenset( + ['pms', 'portage-1', 'portage-2']) + +_portage1_profiles_allow_directories = frozenset( + ["portage-1-compat", "portage-1", 'portage-2']) + _repo_name_sub_re = re.compile(r'[^\w-]') def _gen_valid_repo(name): @@ -49,9 +55,9 @@ class RepoConfig(object): 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi', 'eclass_db', 'eclass_locations', 'eclass_overrides', 'format', 'location', 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name', - 'name', 'priority', 'sign_commit', 'sign_manifest', 'sync', 'thin_manifest', - 'update_changelog', 'user_location', 'portage1_profiles', - 'portage1_profiles_compat') + 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority', + 'profile_formats', 'sign_commit', 'sign_manifest', 'sync', + 'thin_manifest', 'update_changelog', 'user_location') def __init__(self, name, repo_opts): """Build a RepoConfig with options in repo_opts @@ -153,10 +159,11 @@ class RepoConfig(object): for value in ('allow-missing-manifest', 'allow-provide-virtual', 'cache-formats', 'create-manifest', 'disable-manifest', 'manifest-hashes', + 'profile-formats', 'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'): setattr(self, value.lower().replace("-", "_"), layout_data[value]) - self.portage1_profiles = any(x.startswith("portage-1") \ + self.portage1_profiles = any(x in _portage1_profiles_allow_directories for x in layout_data['profile-formats']) self.portage1_profiles_compat = layout_data['profile-formats'] == ('portage-1-compat',) @@ -483,13 +490,6 @@ class RepoConfigLoader(object): prepos_order = [repo.name for (key, repo) in prepos_order if repo.name == key and repo.location is not None] - if portdir in location_map: - portdir_repo = prepos[location_map[portdir]] - portdir_sync = settings.get('SYNC', '') - #if SYNC variable is set and not overwritten by repos.conf - if portdir_sync and not portdir_repo.sync: - portdir_repo.sync = portdir_sync - if prepos['DEFAULT'].main_repo is None or \ prepos['DEFAULT'].main_repo not in prepos: #setting main_repo if it was not set in repos.conf @@ -712,10 +712,10 @@ def parse_layout_conf(repo_location, repo_name=None): # for compatibility w/ PMS, fallback to pms; but also check if the # cache exists or not. - cache_formats = layout_data.get('cache-formats', 'pms').lower().split() - if 'pms' in cache_formats and not os.path.isdir( + cache_formats = layout_data.get('cache-formats', '').lower().split() + if not cache_formats and os.path.isdir( os.path.join(repo_location, 'metadata', 'cache')): - cache_formats.remove('pms') + cache_formats = ['pms'] data['cache-formats'] = tuple(cache_formats) manifest_hashes = layout_data.get('manifest-hashes') @@ -760,7 +760,7 @@ def parse_layout_conf(repo_location, repo_name=None): raw_formats = ('portage-1-compat',) else: raw_formats = set(raw_formats.split()) - unknown = raw_formats.difference(['pms', 'portage-1']) + unknown = raw_formats.difference(_valid_profile_formats) if unknown: repo_name = _get_repo_name(repo_location, cached=repo_name) warnings.warn((_("Repository named '%(repo_name)s' has unsupported " @@ -770,7 +770,7 @@ def parse_layout_conf(repo_location, repo_name=None): layout_filename=layout_filename, unknown_fmts=" ".join(unknown))), DeprecationWarning) - raw_formats = tuple(raw_formats.intersection(['pms', 'portage-1'])) + raw_formats = tuple(raw_formats.intersection(_valid_profile_formats)) data['profile-formats'] = raw_formats return data, layout_errors diff --git a/pym/portage/tests/dbapi/test_fakedbapi.py b/pym/portage/tests/dbapi/test_fakedbapi.py index bce824530fa..e3843f0a448 100644 --- a/pym/portage/tests/dbapi/test_fakedbapi.py +++ b/pym/portage/tests/dbapi/test_fakedbapi.py @@ -1,4 +1,4 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import tempfile @@ -53,6 +53,9 @@ class TestFakedbapi(TestCase): fakedb.cpv_inject(cpv, metadata=metadata) for atom, expected_result in match_tests: - self.assertEqual( fakedb.match(atom), expected_result ) + result = fakedb.match(atom) + self.assertEqual(fakedb.match(atom), expected_result, + "fakedb.match('%s') = %s != %s" % + (atom, result, expected_result)) finally: shutil.rmtree(tempdir) diff --git a/pym/portage/tests/dep/testAtom.py b/pym/portage/tests/dep/testAtom.py index 092cacf84c0..f5a7d374971 100644 --- a/pym/portage/tests/dep/testAtom.py +++ b/pym/portage/tests/dep/testAtom.py @@ -20,6 +20,10 @@ class TestAtom(TestCase): (None, 'sys-apps/portage', None, '0', '[doc]', None), False, False ), ( "*/*", (None, '*/*', None, None, None, None), True, False ), + ( "=*/*-*9999*", + ('=*', '*/*', '*9999*', None, None, None), True, False ), + ( "=*/*-*9999*:0::repo_name", + ('=*', '*/*', '*9999*', '0', None, 'repo_name'), True, True ), ( "sys-apps/*", (None, 'sys-apps/*', None, None, None, None), True, False ), ( "*/portage", @@ -144,6 +148,25 @@ class TestAtom(TestCase): self.assertRaisesMsg(atom, (InvalidAtom, TypeError), Atom, atom, \ allow_wildcard=allow_wildcard, allow_repo=allow_repo) + def testSlotAbiAtom(self): + tests = ( + ("virtual/ffmpeg:0/53", "4-slot-abi", {"slot": "0", "slot_abi": "53", "slot_abi_op": None}), + ("virtual/ffmpeg:0/53=", "4-slot-abi", {"slot": "0", "slot_abi": "53", "slot_abi_op": "="}), + ("virtual/ffmpeg:=", "4-slot-abi", {"slot": None, "slot_abi": None, "slot_abi_op": "="}), + ("virtual/ffmpeg:0=", "4-slot-abi", {"slot": "0", "slot_abi": None, "slot_abi_op": "="}), + ("virtual/ffmpeg:*", "4-slot-abi", {"slot": None, "slot_abi": None, "slot_abi_op": "*"}), + ("virtual/ffmpeg:0*", "4-slot-abi", {"slot": "0", "slot_abi": None, "slot_abi_op": "*"}), + ("virtual/ffmpeg:0", "4-slot-abi", {"slot": "0", "slot_abi": None, "slot_abi_op": None}), + ("virtual/ffmpeg", "4-slot-abi", {"slot": None, "slot_abi": None, "slot_abi_op": None}), + ) + + for atom, eapi, parts in tests: + a = Atom(atom, eapi=eapi) + for k, v in parts.items(): + self.assertEqual(v, getattr(a, k), + msg="Atom('%s').%s = %s == '%s'" % + (atom, k, getattr(a, k), v )) + def test_intersects(self): test_cases = ( ("dev-libs/A", "dev-libs/A", True), diff --git a/pym/portage/tests/dep/testStandalone.py b/pym/portage/tests/dep/testStandalone.py index e9f01df03de..f03f2d508cf 100644 --- a/pym/portage/tests/dep/testStandalone.py +++ b/pym/portage/tests/dep/testStandalone.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -29,7 +29,8 @@ class TestStandalone(TestCase): ) for cpv1, cpv2, expected_result in test_cases: - self.assertEqual(cpvequal(cpv1, cpv2), expected_result) + self.assertEqual(cpvequal(cpv1, cpv2), expected_result, + "cpvequal('%s', '%s') != %s" % (cpv1, cpv2, expected_result)) for cpv1, cpv2 in test_cases_xfail: self.assertRaisesMsg("cpvequal("+cpv1+", "+cpv2+")", \ diff --git a/pym/portage/tests/dep/test_best_match_to_list.py b/pym/portage/tests/dep/test_best_match_to_list.py index 58ab95b0fa7..8a14038280b 100644 --- a/pym/portage/tests/dep/test_best_match_to_list.py +++ b/pym/portage/tests/dep/test_best_match_to_list.py @@ -38,6 +38,8 @@ class Test_best_match_to_list(TestCase): [Atom("=dev-libs/A-1:0")], True), ("dev-libs/A-1", [Atom("dev-libs/*", allow_wildcard=True), Atom("=dev-libs/A-1:0")], \ [Atom("=dev-libs/A-1:0"), Atom("dev-libs/*", allow_wildcard=True)], True), + ("dev-libs/A-4.9999-r1", [Atom("dev-libs/*", allow_wildcard=True), Atom("=*/*-*9999*", allow_wildcard=True)], \ + [Atom("=*/*-*9999*", allow_wildcard=True), Atom("dev-libs/*", allow_wildcard=True)], True), ("dev-libs/A-1:0", [Atom("dev-*/*", allow_wildcard=True), Atom("dev-*/*:0", allow_wildcard=True),\ Atom("dev-libs/A"), Atom("<=dev-libs/A-2"), Atom("dev-libs/A:0"), \ Atom("=dev-libs/A-1*"), Atom("~dev-libs/A-1"), Atom("=dev-libs/A-1")], \ diff --git a/pym/portage/tests/dep/test_isvalidatom.py b/pym/portage/tests/dep/test_isvalidatom.py index 173ab0decb9..abcec755e87 100644 --- a/pym/portage/tests/dep/test_isvalidatom.py +++ b/pym/portage/tests/dep/test_isvalidatom.py @@ -134,6 +134,15 @@ class IsValidAtom(TestCase): IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[bar?,!baz?,!doc=,build=]", False, allow_repo=False), IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[doc?]", False, allow_repo=False), IsValidAtomTestCase("null/portage::repo", False, allow_repo=False), + + IsValidAtomTestCase("virtual/ffmpeg:0/53", True), + IsValidAtomTestCase("virtual/ffmpeg:0/53=", True), + IsValidAtomTestCase("virtual/ffmpeg:0/53*", False), + IsValidAtomTestCase("virtual/ffmpeg:=", True), + IsValidAtomTestCase("virtual/ffmpeg:0=", True), + IsValidAtomTestCase("virtual/ffmpeg:*", True), + IsValidAtomTestCase("virtual/ffmpeg:0*", True), + IsValidAtomTestCase("virtual/ffmpeg:0", True), ) for test_case in test_cases: diff --git a/pym/portage/tests/dep/test_match_from_list.py b/pym/portage/tests/dep/test_match_from_list.py index afba4141f06..d5d718f74a1 100644 --- a/pym/portage/tests/dep/test_match_from_list.py +++ b/pym/portage/tests/dep/test_match_from_list.py @@ -1,10 +1,10 @@ -# Copyright 2006, 2010 Gentoo Foundation +# Copyright 2006-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys from portage.tests import TestCase from portage.dep import Atom, match_from_list, _repo_separator -from portage.versions import catpkgsplit +from portage.versions import catpkgsplit, _pkg_str if sys.hexversion >= 0x3000000: basestring = str @@ -16,9 +16,15 @@ class Package(object): def __init__(self, atom): atom = Atom(atom, allow_repo=True) self.cp = atom.cp - self.cpv = atom.cpv + slot = atom.slot + if atom.slot_abi: + slot = "%s/%s" % (slot, atom.slot_abi) + if not slot: + slot = '0' + self.cpv = _pkg_str(atom.cpv, slot=slot, repo=atom.repo) self.cpv_split = catpkgsplit(self.cpv) - self.slot = atom.slot + self.slot = self.cpv.slot + self.slot_abi = self.cpv.slot_abi self.repo = atom.repo if atom.use: self.use = self._use_class(atom.use.enabled) @@ -68,6 +74,8 @@ class Test_match_from_list(TestCase): ("=sys-fs/udev-1*", ["sys-fs/udev-123"], ["sys-fs/udev-123"]), ("=sys-fs/udev-4*", ["sys-fs/udev-456"], ["sys-fs/udev-456"] ), ("*/*", ["sys-fs/udev-456"], ["sys-fs/udev-456"] ), + ("*/*:0", ["sys-fs/udev-456:0"], ["sys-fs/udev-456:0"] ), + ("*/*:1", ["sys-fs/udev-456:0"], [] ), ("sys-fs/*", ["sys-fs/udev-456"], ["sys-fs/udev-456"] ), ("*/udev", ["sys-fs/udev-456"], ["sys-fs/udev-456"] ), ("=sys-apps/portage-2*", ["sys-apps/portage-2.1"], ["sys-apps/portage-2.1"] ), @@ -93,6 +101,23 @@ class Test_match_from_list(TestCase): ("dev-libs/A::repo2[foo]", [Package("=dev-libs/A-1::repo1[-foo]"), Package("=dev-libs/A-1::repo2[foo]")], ["dev-libs/A-1::repo2"] ), ("dev-libs/A:1::repo2[foo]", [Package("=dev-libs/A-1:1::repo1"), Package("=dev-libs/A-1:2::repo2")], [] ), ("dev-libs/A:1::repo2[foo]", [Package("=dev-libs/A-1:2::repo1"), Package("=dev-libs/A-1:1::repo2[foo]")], ["dev-libs/A-1::repo2"] ), + + ("virtual/ffmpeg:0/53", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0/53=", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0/52", [Package("=virtual/ffmpeg-0.10.3:0/53")], [] ), + ("virtual/ffmpeg:=", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0=", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:*", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0*", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + ("virtual/ffmpeg:0", [Package("=virtual/ffmpeg-0.10.3:0/53")], ["virtual/ffmpeg-0.10.3"] ), + + ("sys-libs/db:4.8/4.8", [Package("=sys-libs/db-4.8.30:4.8")], ["sys-libs/db-4.8.30"] ), + ("sys-libs/db:4.8/4.8=", [Package("=sys-libs/db-4.8.30:4.8")], ["sys-libs/db-4.8.30"] ), + ("sys-libs/db:4.8=", [Package("=sys-libs/db-4.8.30:4.8")], ["sys-libs/db-4.8.30"] ), + ("sys-libs/db:4.8*", [Package("=sys-libs/db-4.8.30:4.8")], ["sys-libs/db-4.8.30"] ), + ("sys-libs/db:*", [Package("=sys-libs/db-4.8.30:4.8")], ["sys-libs/db-4.8.30"] ), + ("sys-libs/db:4.8/0", [Package("=sys-libs/db-4.8.30:4.8")], [] ), + ("sys-libs/db:4.8/0=", [Package("=sys-libs/db-4.8.30:4.8")], [] ), ) for atom, cpv_list, expected_result in tests: diff --git a/pym/portage/tests/ebuild/test_config.py b/pym/portage/tests/ebuild/test_config.py index a1ba202d54d..63cb99d41ce 100644 --- a/pym/portage/tests/ebuild/test_config.py +++ b/pym/portage/tests/ebuild/test_config.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage @@ -220,7 +220,7 @@ class ConfigTestCase(TestCase): ( "profile-formats = pms", "thin-manifests = true", - "manifest-hashes = RMD160 SHA1 SHA256 SHA512 WHIRLPOOL", + "manifest-hashes = SHA256 SHA512 WHIRLPOOL", ), } } diff --git a/pym/portage/tests/emerge/test_emerge_slot_abi.py b/pym/portage/tests/emerge/test_emerge_slot_abi.py new file mode 100644 index 00000000000..f18bd123b63 --- /dev/null +++ b/pym/portage/tests/emerge/test_emerge_slot_abi.py @@ -0,0 +1,201 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import subprocess +import sys + +import portage +from portage import os +from portage import _unicode_decode +from portage.const import (BASH_BINARY, PORTAGE_BIN_PATH, + PORTAGE_PYM_PATH, USER_CONFIG_PATH) +from portage.process import find_binary +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.util import ensure_dirs + +class SlotAbiEmergeTestCase(TestCase): + + def testSlotAbiEmerge(self): + + debug = False + + ebuilds = { + "dev-libs/glib-1.2.10" : { + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/glib-2.32.3" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.32" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2=", + "RDEPEND": "dev-libs/glib:2=" + }, + } + installed = { + "dev-libs/glib-1.2.10" : { + "EAPI": "4-slot-abi", + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2/2.30=", + "RDEPEND": "dev-libs/glib:2/2.30=" + }, + } + + world = ["dev-libs/glib:1", "dev-libs/dbus-glib"] + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=debug) + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + trees = playground.trees + portdb = trees[eroot]["porttree"].dbapi + vardb = trees[eroot]["vartree"].dbapi + portdir = settings["PORTDIR"] + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH) + package_mask_path = os.path.join(user_config_dir, "package.mask") + + portage_python = portage._python_interpreter + ebuild_cmd = (portage_python, "-Wd", + os.path.join(PORTAGE_BIN_PATH, "ebuild")) + emerge_cmd = (portage_python, "-Wd", + os.path.join(PORTAGE_BIN_PATH, "emerge")) + + test_ebuild = portdb.findname("dev-libs/dbus-glib-0.98") + self.assertFalse(test_ebuild is None) + + test_commands = ( + emerge_cmd + ("--oneshot", "dev-libs/glib",), + (lambda: "dev-libs/glib:2/2.32=" in vardb.aux_get("dev-libs/dbus-glib-0.98", ["RDEPEND"])[0],), + (BASH_BINARY, "-c", "echo %s >> %s" % + tuple(map(portage._shell_quote, + (">=dev-libs/glib-2.32", package_mask_path,)))), + emerge_cmd + ("--oneshot", "dev-libs/glib",), + (lambda: "dev-libs/glib:2/2.30=" in vardb.aux_get("dev-libs/dbus-glib-0.98", ["RDEPEND"])[0],), + ) + + distdir = playground.distdir + pkgdir = playground.pkgdir + fake_bin = os.path.join(eprefix, "bin") + portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") + profile_path = settings.profile_path + + features = [] + if not portage.process.sandbox_capable or \ + os.environ.get("SANDBOX_ON") == "1": + features.append("-sandbox") + + make_conf = ( + "FEATURES=\"%s\"\n" % (" ".join(features),), + "PORTDIR=\"%s\"\n" % (portdir,), + "PORTAGE_GRPNAME=\"%s\"\n" % (os.environ["PORTAGE_GRPNAME"],), + "PORTAGE_USERNAME=\"%s\"\n" % (os.environ["PORTAGE_USERNAME"],), + "PKGDIR=\"%s\"\n" % (pkgdir,), + "PORTAGE_INST_GID=%s\n" % (portage.data.portage_gid,), + "PORTAGE_INST_UID=%s\n" % (portage.data.portage_uid,), + "PORTAGE_TMPDIR=\"%s\"\n" % (portage_tmpdir,), + "CLEAN_DELAY=0\n", + "DISTDIR=\"%s\"\n" % (distdir,), + "EMERGE_WARNING_DELAY=0\n", + ) + + path = os.environ.get("PATH") + if path is not None and not path.strip(): + path = None + if path is None: + path = "" + else: + path = ":" + path + path = fake_bin + path + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath is not None and not pythonpath.strip(): + pythonpath = None + if pythonpath is not None and \ + pythonpath.split(":")[0] == PORTAGE_PYM_PATH: + pass + else: + if pythonpath is None: + pythonpath = "" + else: + pythonpath = ":" + pythonpath + pythonpath = PORTAGE_PYM_PATH + pythonpath + + env = { + "PORTAGE_OVERRIDE_EPREFIX" : eprefix, + "PATH" : path, + "PORTAGE_PYTHON" : portage_python, + "PYTHONPATH" : pythonpath, + } + + if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: + env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ + os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] + + dirs = [distdir, fake_bin, portage_tmpdir, + user_config_dir, var_cache_edb] + true_symlinks = ["chown", "chgrp"] + true_binary = find_binary("true") + self.assertEqual(true_binary is None, False, + "true command not found") + try: + for d in dirs: + ensure_dirs(d) + with open(os.path.join(user_config_dir, "make.conf"), 'w') as f: + for line in make_conf: + f.write(line) + for x in true_symlinks: + os.symlink(true_binary, os.path.join(fake_bin, x)) + with open(os.path.join(var_cache_edb, "counter"), 'wb') as f: + f.write(b"100") + # non-empty system set keeps --depclean quiet + with open(os.path.join(profile_path, "packages"), 'w') as f: + f.write("*dev-libs/token-system-pkg") + + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + for i, args in enumerate(test_commands): + + if hasattr(args[0], '__call__'): + self.assertTrue(args[0](), + "callable at index %s failed" % (i,)) + continue + + proc = subprocess.Popen(args, + env=env, stdout=stdout) + + if debug: + proc.wait() + else: + output = proc.stdout.readlines() + proc.wait() + proc.stdout.close() + if proc.returncode != os.EX_OK: + for line in output: + sys.stderr.write(_unicode_decode(line)) + + self.assertEqual(os.EX_OK, proc.returncode, + "emerge failed with args %s" % (args,)) + finally: + playground.cleanup() diff --git a/pym/portage/tests/emerge/test_global_updates.py b/pym/portage/tests/emerge/test_global_updates.py index 73556210016..eb54310593f 100644 --- a/pym/portage/tests/emerge/test_global_updates.py +++ b/pym/portage/tests/emerge/test_global_updates.py @@ -1,4 +1,4 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -15,6 +15,8 @@ slotmove invalid_atom 0 3 slotmove !=invalid/blocker-3* 0 3 slotmove =valid/atom-3* 0 3 invalid_extra_token slotmove =valid/atom-3* 0 3 +slotmove =valid/atom-3* 0 3/3.1 +slotmove =valid/atom-3* 0/0 3 move valid/atom1 valid/atom2 invalid_extra_token move valid/atom1 invalid_atom2 move invalid_atom1 valid/atom2 @@ -28,7 +30,7 @@ move valid/atom1 valid/atom2 ['slotmove', Atom('=valid/atom-3*'), '0', '3'], ['move', Atom('valid/atom1'), Atom('valid/atom2')], ], - 10, + 12, ), ) diff --git a/pym/portage/tests/emerge/test_simple.py b/pym/portage/tests/emerge/test_simple.py index 28ac098430f..f87170a71f3 100644 --- a/pym/portage/tests/emerge/test_simple.py +++ b/pym/portage/tests/emerge/test_simple.py @@ -1,4 +1,4 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import subprocess @@ -259,8 +259,8 @@ pkg_preinst() { emerge_cmd + ("-C", "--quiet", "dev-libs/B"), ) - distdir = os.path.join(eprefix, "distdir") - pkgdir = os.path.join(eprefix, "pkgdir") + distdir = playground.distdir + pkgdir = playground.pkgdir fake_bin = os.path.join(eprefix, "bin") portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") profile_path = settings.profile_path diff --git a/pym/portage/tests/repoman/test_echangelog.py b/pym/portage/tests/repoman/test_echangelog.py new file mode 100644 index 00000000000..502aa72922c --- /dev/null +++ b/pym/portage/tests/repoman/test_echangelog.py @@ -0,0 +1,110 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import datetime +import subprocess +import sys +import tempfile +import time + +import portage +from portage import os +from portage import shutil +from portage.tests import TestCase +from repoman.utilities import UpdateChangeLog + +class RepomanEchangelogTestCase(TestCase): + + def setUp(self): + super(RepomanEchangelogTestCase, self).setUp() + + self.tmpdir = tempfile.mkdtemp(prefix='repoman.echangelog.') + + self.skel_changelog = os.path.join(self.tmpdir, 'skel.ChangeLog') + skel = [ + '# ChangeLog for <CATEGORY>/<PACKAGE_NAME>\n', + '# Copyright 1999-2000 Gentoo Foundation; Distributed under the GPL v2\n', + '# $Header: $\n' + ] + self._writelines(self.skel_changelog, skel) + + self.cat = 'mycat' + self.pkg = 'mypkg' + self.pkgdir = os.path.join(self.tmpdir, self.cat, self.pkg) + os.makedirs(self.pkgdir) + + self.header_pkg = '# ChangeLog for %s/%s\n' % (self.cat, self.pkg) + self.header_copyright = '# Copyright 1999-%s Gentoo Foundation; Distributed under the GPL v2\n' % \ + datetime.datetime.now().year + self.header_cvs = '# $Header: $\n' + + self.changelog = os.path.join(self.pkgdir, 'ChangeLog') + + self.user = 'Testing User <portage@gentoo.org>' + + def tearDown(self): + super(RepomanEchangelogTestCase, self).tearDown() + shutil.rmtree(self.tmpdir) + + def _readlines(self, file): + with open(file, 'r') as f: + return f.readlines() + + def _writelines(self, file, data): + with open(file, 'w') as f: + f.writelines(data) + + def testRejectRootUser(self): + self.assertEqual(UpdateChangeLog(self.pkgdir, 'me <root@gentoo.org>', '', '', '', '', quiet=True), None) + + def testMissingSkelFile(self): + # Test missing ChangeLog, but with empty skel (i.e. do nothing). + UpdateChangeLog(self.pkgdir, self.user, 'test!', '/does/not/exist', self.cat, self.pkg, quiet=True) + actual_cl = self._readlines(self.changelog) + self.assertTrue(len(actual_cl[0]) > 0) + + def testEmptyChangeLog(self): + # Make sure we do the right thing with a 0-byte ChangeLog + open(self.changelog, 'w').close() + UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) + actual_cl = self._readlines(self.changelog) + self.assertEqual(actual_cl[0], self.header_pkg) + self.assertEqual(actual_cl[1], self.header_copyright) + self.assertEqual(actual_cl[2], self.header_cvs) + + def testCopyrightUpdate(self): + # Make sure updating the copyright line works + UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) + actual_cl = self._readlines(self.changelog) + self.assertEqual(actual_cl[1], self.header_copyright) + + def testSkelHeader(self): + # Test skel.ChangeLog -> ChangeLog + UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) + actual_cl = self._readlines(self.changelog) + self.assertEqual(actual_cl[0], self.header_pkg) + self.assertNotEqual(actual_cl[-1], '\n') + + def testExistingGoodHeader(self): + # Test existing ChangeLog (correct values) + self._writelines(self.changelog, [self.header_pkg]) + + UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) + actual_cl = self._readlines(self.changelog) + self.assertEqual(actual_cl[0], self.header_pkg) + + def testExistingBadHeader(self): + # Test existing ChangeLog (wrong values) + self._writelines(self.changelog, ['# ChangeLog for \n']) + + UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) + actual_cl = self._readlines(self.changelog) + self.assertEqual(actual_cl[0], self.header_pkg) + + def testTrailingNewlines(self): + # Make sure trailing newlines get chomped. + self._writelines(self.changelog, ['#\n', 'foo\n', '\n', 'bar\n', '\n', '\n']) + + UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) + actual_cl = self._readlines(self.changelog) + self.assertNotEqual(actual_cl[-1], '\n') diff --git a/pym/portage/tests/resolver/ResolverPlayground.py b/pym/portage/tests/resolver/ResolverPlayground.py index 140b25d6e4f..0ac209761a8 100644 --- a/pym/portage/tests/resolver/ResolverPlayground.py +++ b/pym/portage/tests/resolver/ResolverPlayground.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from itertools import permutations @@ -9,9 +9,6 @@ from portage import os from portage import shutil from portage.const import (GLOBAL_CONFIG_PATH, PORTAGE_BASE_PATH, USER_CONFIG_PATH) -from portage.dbapi.vartree import vartree -from portage.dbapi.porttree import portagetree -from portage.dbapi.bintree import binarytree from portage.dep import Atom, _repo_separator from portage.package.ebuild.config import config from portage.package.ebuild.digestgen import digestgen @@ -56,7 +53,7 @@ class ResolverPlayground(object): </pkgmetadata> """ - def __init__(self, ebuilds={}, installed={}, profile={}, repo_configs={}, \ + def __init__(self, ebuilds={}, binpkgs={}, installed={}, profile={}, repo_configs={}, \ user_config={}, sets={}, world=[], world_sets=[], distfiles={}, debug=False): """ ebuilds: cpv -> metadata mapping simulating available ebuilds. @@ -68,6 +65,7 @@ class ResolverPlayground(object): self.eprefix = normalize_path(tempfile.mkdtemp()) self.eroot = self.eprefix + os.sep self.distdir = os.path.join(self.eroot, "var", "portage", "distfiles") + self.pkgdir = os.path.join(self.eprefix, "pkgdir") self.portdir = os.path.join(self.eroot, "usr/portage") self.vdbdir = os.path.join(self.eroot, "var/db/pkg") os.makedirs(self.portdir) @@ -82,6 +80,7 @@ class ResolverPlayground(object): self._create_distfiles(distfiles) self._create_ebuilds(ebuilds) + self._create_binpkgs(binpkgs) self._create_installed(installed) self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets) self._create_world(world, world_sets) @@ -205,6 +204,29 @@ class ResolverPlayground(object): if not digestgen(mysettings=tmpsettings, myportdb=portdb): raise AssertionError('digest creation failed for %s' % ebuild_path) + def _create_binpkgs(self, binpkgs): + for cpv, metadata in binpkgs.items(): + a = Atom("=" + cpv, allow_repo=True) + repo = a.repo + if repo is None: + repo = "test_repo" + + cat, pf = catsplit(a.cpv) + metadata = metadata.copy() + metadata.setdefault("SLOT", "0") + metadata.setdefault("KEYWORDS", "x86") + metadata.setdefault("BUILD_TIME", "0") + metadata["repository"] = repo + metadata["CATEGORY"] = cat + metadata["PF"] = pf + + repo_dir = self.pkgdir + category_dir = os.path.join(repo_dir, cat) + binpkg_path = os.path.join(category_dir, pf + ".tbz2") + ensure_dirs(category_dir) + t = portage.xpak.tbz2(binpkg_path) + t.recompose_mem(portage.xpak.xpak_mem(metadata)) + def _create_installed(self, installed): for cpv in installed: a = Atom("=" + cpv, allow_repo=True) @@ -223,6 +245,7 @@ class ResolverPlayground(object): lic = metadata.pop("LICENSE", "") properties = metadata.pop("PROPERTIES", "") slot = metadata.pop("SLOT", 0) + build_time = metadata.pop("BUILD_TIME", "0") keywords = metadata.pop("KEYWORDS", "~x86") iuse = metadata.pop("IUSE", "") use = metadata.pop("USE", "") @@ -241,6 +264,7 @@ class ResolverPlayground(object): f.close() write_key("EAPI", eapi) + write_key("BUILD_TIME", build_time) write_key("COUNTER", "0") write_key("LICENSE", lic) write_key("PROPERTIES", properties) @@ -473,12 +497,15 @@ class ResolverPlayground(object): env = { "ACCEPT_KEYWORDS": "x86", "DISTDIR" : self.distdir, - "PKGDIR": os.path.join(self.eroot, "usr/portage/packages"), + "PKGDIR": self.pkgdir, "PORTDIR": self.portdir, "PORTDIR_OVERLAY": " ".join(portdir_overlay), 'PORTAGE_TMPDIR' : os.path.join(self.eroot, 'var/tmp'), } + if os.environ.get("NOCOLOR"): + env["NOCOLOR"] = os.environ["NOCOLOR"] + if os.environ.get("SANDBOX_ON") == "1": # avoid problems from nested sandbox instances env["FEATURES"] = "-sandbox" @@ -511,6 +538,9 @@ class ResolverPlayground(object): elif options.get("--prune"): action = "prune" + if "--usepkgonly" in options: + options["--usepkg"] = True + global_noiselimit = portage.util.noiselimit global_emergelog_disable = _emerge.emergelog._disable try: @@ -599,8 +629,7 @@ class ResolverPlaygroundTestCase(object): if cpv[:1] == "!": new_got.append(cpv) continue - a = Atom("="+cpv, allow_repo=True) - new_got.append(a.cpv) + new_got.append(cpv.split(_repo_separator)[0]) got = new_got if expected: new_expected = [] @@ -609,13 +638,13 @@ class ResolverPlaygroundTestCase(object): if obj[:1] == "!": new_expected.append(obj) continue - a = Atom("="+obj, allow_repo=True) - new_expected.append(a.cpv) + new_expected.append( + obj.split(_repo_separator)[0]) continue new_expected.append(set()) for cpv in obj: if cpv[:1] != "!": - cpv = Atom("="+cpv, allow_repo=True).cpv + cpv = cpv.split(_repo_separator)[0] new_expected[-1].add(cpv) expected = new_expected if self.ignore_mergelist_order and got is not None: @@ -720,7 +749,14 @@ class ResolverPlaygroundResult(object): repo_str = "" if x.metadata["repository"] != "test_repo": repo_str = _repo_separator + x.metadata["repository"] - self.mergelist.append(x.cpv + repo_str) + mergelist_str = x.cpv + repo_str + if x.built: + if x.operation == "merge": + desc = x.type_name + else: + desc = x.operation + mergelist_str = "[%s]%s" % (desc, mergelist_str) + self.mergelist.append(mergelist_str) if self.depgraph._dynamic_config._needed_use_config_changes: self.use_changes = {} diff --git a/pym/portage/tests/resolver/test_autounmask.py b/pym/portage/tests/resolver/test_autounmask.py index 46dbab18e0b..6acac9984aa 100644 --- a/pym/portage/tests/resolver/test_autounmask.py +++ b/pym/portage/tests/resolver/test_autounmask.py @@ -1,7 +1,6 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from portage.const import _ENABLE_SET_CONFIG from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase @@ -285,6 +284,9 @@ class AutounmaskTestCase(TestCase): "dev-libs/E-1": { "LICENSE": "TEST" }, "dev-libs/E-2": { "LICENSE": "TEST" }, "dev-libs/F-1": { "DEPEND": "=dev-libs/E-1", "LICENSE": "TEST" }, + + "dev-java/sun-jdk-1.6.0.32": { "LICENSE": "TEST", "KEYWORDS": "~x86" }, + "dev-java/sun-jdk-1.6.0.31": { "LICENSE": "TEST", "KEYWORDS": "x86" }, } test_cases = ( @@ -316,6 +318,14 @@ class AutounmaskTestCase(TestCase): success = False, mergelist = ["dev-libs/E-1", "dev-libs/F-1", "dev-libs/D-1"], license_changes = { "dev-libs/D-1": set(["TEST"]), "dev-libs/E-1": set(["TEST"]), "dev-libs/E-2": set(["TEST"]), "dev-libs/F-1": set(["TEST"]) }), + + #Test license only for bug #420847 + ResolverPlaygroundTestCase( + ["dev-java/sun-jdk"], + options = {"--autounmask": True}, + success = False, + mergelist = ["dev-java/sun-jdk-1.6.0.31"], + license_changes = { "dev-java/sun-jdk-1.6.0.31": set(["TEST"]) }), ) playground = ResolverPlayground(ebuilds=ebuilds) @@ -329,9 +339,6 @@ class AutounmaskTestCase(TestCase): def testAutounmaskAndSets(self): - if not _ENABLE_SET_CONFIG: - return - ebuilds = { #ebuilds to test use changes "dev-libs/A-1": { }, diff --git a/pym/portage/tests/resolver/test_complete_graph.py b/pym/portage/tests/resolver/test_complete_graph.py index 07206430e77..1b0342c673a 100644 --- a/pym/portage/tests/resolver/test_complete_graph.py +++ b/pym/portage/tests/resolver/test_complete_graph.py @@ -1,4 +1,4 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -7,6 +7,70 @@ from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, class CompleteGraphTestCase(TestCase): + def testCompleteGraphUseChange(self): + """ + Prevent reverse dependency breakage triggered by USE changes. + """ + + ebuilds = { + "dev-libs/libxml2-2.8.0": { + "EAPI": "2", + "IUSE": "+icu", + "SLOT": "2", + }, + "x11-libs/qt-webkit-4.8.2": { + "EAPI": "2", + "IUSE": "icu", + "RDEPEND" : "dev-libs/libxml2:2[!icu?]", + }, + } + + installed = { + "dev-libs/libxml2-2.8.0": { + "EAPI": "2", + "IUSE": "+icu", + "USE": "", + "SLOT": "2", + }, + "x11-libs/qt-webkit-4.8.2": { + "EAPI": "2", + "IUSE": "icu", + "RDEPEND" : "dev-libs/libxml2:2[-icu]", + "USE": "", + } + } + + world = ["x11-libs/qt-webkit"] + + test_cases = ( + + ResolverPlaygroundTestCase( + ["dev-libs/libxml2"], + options = {"--complete-graph-if-new-use" : "y" }, + mergelist = ["dev-libs/libxml2-2.8.0"], + slot_collision_solutions = [{'dev-libs/libxml2-2.8.0': {'icu': False}}], + success = False, + ), + + ResolverPlaygroundTestCase( + ["dev-libs/libxml2"], + options = {"--complete-graph-if-new-use" : "n" }, + mergelist = ["dev-libs/libxml2-2.8.0"], + success = True, + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() + def testCompleteGraphVersionChange(self): """ Prevent reverse dependency breakage triggered by version changes. @@ -29,7 +93,7 @@ class CompleteGraphTestCase(TestCase): test_cases = ( ResolverPlaygroundTestCase( [">=sys-libs/x-2"], - options = {"--complete-graph-if-new-ver" : "n"}, + options = {"--complete-graph-if-new-ver" : "n", "--rebuild-if-new-slot-abi": "n"}, mergelist = ["sys-libs/x-2"], success = True, ), @@ -42,7 +106,7 @@ class CompleteGraphTestCase(TestCase): ), ResolverPlaygroundTestCase( ["<sys-libs/x-1"], - options = {"--complete-graph-if-new-ver" : "n"}, + options = {"--complete-graph-if-new-ver" : "n", "--rebuild-if-new-slot-abi": "n"}, mergelist = ["sys-libs/x-0.1"], success = True, ), diff --git a/pym/portage/tests/resolver/test_merge_order.py b/pym/portage/tests/resolver/test_merge_order.py index adff84d2bf0..5b5709afe17 100644 --- a/pym/portage/tests/resolver/test_merge_order.py +++ b/pym/portage/tests/resolver/test_merge_order.py @@ -376,7 +376,7 @@ class MergeOrderTestCase(TestCase): success = True, all_permutations = True, ambiguous_merge_order = True, - mergelist = [("app-misc/blocker-runtime-a-1", "app-misc/blocker-runtime-b-1"), "app-misc/installed-blocker-a-1", ("!app-misc/blocker-runtime-a", "!app-misc/blocker-runtime-b")]), + mergelist = [("app-misc/blocker-runtime-a-1", "app-misc/blocker-runtime-b-1"), "[uninstall]app-misc/installed-blocker-a-1", ("!app-misc/blocker-runtime-a", "!app-misc/blocker-runtime-b")]), # We have a soft buildtime blocker against an installed # package that should cause it to be uninstalled. Note that with # soft blockers, the blocking packages are allowed to temporarily @@ -386,7 +386,7 @@ class MergeOrderTestCase(TestCase): ResolverPlaygroundTestCase( ["app-misc/blocker-buildtime-unbuilt-a"], success = True, - mergelist = ["app-misc/blocker-buildtime-unbuilt-a-1", "app-misc/installed-blocker-a-1", "!app-misc/installed-blocker-a"]), + mergelist = ["app-misc/blocker-buildtime-unbuilt-a-1", "[uninstall]app-misc/installed-blocker-a-1", "!app-misc/installed-blocker-a"]), # We have a hard buildtime blocker against an installed # package that will not resolve automatically (unless # the option requested in bug 250286 is implemented). @@ -409,7 +409,7 @@ class MergeOrderTestCase(TestCase): ResolverPlaygroundTestCase( ["media-video/libav"], success=True, - mergelist = ['media-video/libav-0.7_pre20110327', 'media-video/ffmpeg-0.7_rc1', '!media-video/ffmpeg']), + mergelist = ['media-video/libav-0.7_pre20110327', '[uninstall]media-video/ffmpeg-0.7_rc1', '!media-video/ffmpeg']), # Test that OS_HEADERS_PACKAGE_ATOM and LIBC_PACKAGE_ATOM # are merged asap, in order to account for implicit # dependencies. See bug #303567. Optimally, satisfied deps diff --git a/pym/portage/tests/resolver/test_simple.py b/pym/portage/tests/resolver/test_simple.py index 0bcfc4b70f0..324ffa2a6c6 100644 --- a/pym/portage/tests/resolver/test_simple.py +++ b/pym/portage/tests/resolver/test_simple.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -17,6 +17,9 @@ class SimpleResolverTestCase(TestCase): "app-misc/X-1": {}, "app-misc/W-1": {}, } + binpkgs = { + "dev-libs/B-1.2": {}, + } installed = { "dev-libs/A-1": {}, "dev-libs/B-1.1": {}, @@ -43,12 +46,26 @@ class SimpleResolverTestCase(TestCase): mergelist = ["dev-libs/B-1.2"]), ResolverPlaygroundTestCase( + ["dev-libs/B"], + options = {"--update": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/B-1.2"]), + + ResolverPlaygroundTestCase( + ["dev-libs/B"], + options = {"--update": True, "--usepkgonly": True}, + success = True, + mergelist = ["[binary]dev-libs/B-1.2"]), + + ResolverPlaygroundTestCase( ["app-misc/Z"], success = True, - mergelist = ["app-misc/W-1", "app-misc/X-1", "app-misc/Z-1"]), + ambiguous_merge_order = True, + mergelist = [("app-misc/W-1", "app-misc/X-1"), "app-misc/Z-1"]), ) - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed) + playground = ResolverPlayground(ebuilds=ebuilds, + binpkgs=binpkgs, installed=installed) try: for test_case in test_cases: playground.run_TestCase(test_case) diff --git a/pym/portage/tests/resolver/test_slot_abi.py b/pym/portage/tests/resolver/test_slot_abi.py new file mode 100644 index 00000000000..6381bcc4ddf --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_abi.py @@ -0,0 +1,376 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SlotAbiTestCase(TestCase): + + def __init__(self, *args, **kwargs): + super(SlotAbiTestCase, self).__init__(*args, **kwargs) + + def testSubSlot(self): + ebuilds = { + "dev-libs/icu-49" : { + "EAPI": "4-slot-abi", + "SLOT": "0/49" + }, + "dev-libs/icu-4.8" : { + "EAPI": "4-slot-abi", + "SLOT": "0/48" + }, + "dev-libs/libxml2-2.7.8" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/icu:=", + "RDEPEND": "dev-libs/icu:=" + }, + } + binpkgs = { + "dev-libs/icu-49" : { + "EAPI": "4-slot-abi", + "SLOT": "0/49" + }, + "dev-libs/icu-4.8" : { + "EAPI": "4-slot-abi", + "SLOT": "0/48" + }, + "dev-libs/libxml2-2.7.8" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/icu:0/48=", + "RDEPEND": "dev-libs/icu:0/48=" + }, + } + installed = { + "dev-libs/icu-4.8" : { + "EAPI": "4-slot-abi", + "SLOT": "0/48" + }, + "dev-libs/libxml2-2.7.8" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/icu:0/48=", + "RDEPEND": "dev-libs/icu:0/48=" + }, + } + + world = ["dev-libs/libxml2"] + + test_cases = ( + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True}, + success = True, + mergelist = ["dev-libs/icu-49", "dev-libs/libxml2-2.7.8" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/icu-49"]), + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/icu-49", "dev-libs/libxml2-2.7.8" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True, "--usepkgonly": True}, + success = True, + mergelist = ["[binary]dev-libs/icu-4.8"]), + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True, "--usepkgonly": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]dev-libs/icu-49"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ["dev-libs/icu-49", "dev-libs/libxml2-2.7.8" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/icu-49"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/icu-49", "dev-libs/libxml2-2.7.8" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True}, + success = True, + mergelist = []), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]dev-libs/icu-49"]), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, binpkgs=binpkgs, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() + + def testWholeSlot(self): + ebuilds = { + "sys-libs/db-4.8" : { + "SLOT": "4.8" + }, + "sys-libs/db-4.7" : { + "SLOT": "4.7" + }, + "app-office/libreoffice-3.5.4.2" : { + "EAPI": "4-slot-abi", + "DEPEND": ">=sys-libs/db-4:=", + "RDEPEND": ">=sys-libs/db-4:=" + }, + } + binpkgs = { + "sys-libs/db-4.8" : { + "SLOT": "4.8" + }, + "sys-libs/db-4.7" : { + "SLOT": "4.7" + }, + "app-office/libreoffice-3.5.4.2" : { + "EAPI": "4-slot-abi", + "DEPEND": ">=sys-libs/db-4:4.7/4.7=", + "RDEPEND": ">=sys-libs/db-4:4.7/4.7=" + }, + } + installed = { + "sys-libs/db-4.7" : { + "SLOT": "4.7" + }, + "app-office/libreoffice-3.5.4.2" : { + "EAPI": "4-slot-abi", + "DEPEND": ">=sys-libs/db-4:4.7/4.7=", + "RDEPEND": ">=sys-libs/db-4:4.7/4.7=" + }, + } + + world = ["app-office/libreoffice"] + + test_cases = ( + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True}, + success = True, + mergelist = ["sys-libs/db-4.8", "app-office/libreoffice-3.5.4.2"]), + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8", "app-office/libreoffice-3.5.4.2"]), + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True, "--usepkgonly": True}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["sys-libs/db"], + options = {"--oneshot": True, "--rebuild-if-new-slot-abi": "n"}, + success = True, + mergelist = ["sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ["sys-libs/db-4.8", "app-office/libreoffice-3.5.4.2"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8", "app-office/libreoffice-3.5.4.2"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True}, + success = True, + mergelist = []), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]sys-libs/db-4.8"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--rebuild-if-new-slot-abi": "n"}, + success = True, + mergelist = []), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, binpkgs=binpkgs, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() + + def testWholeSlotSubSlotMix(self): + ebuilds = { + "dev-libs/glib-1.2.10" : { + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/glib-2.32.3" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.32" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2=", + "RDEPEND": "dev-libs/glib:2=" + }, + } + binpkgs = { + "dev-libs/glib-1.2.10" : { + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/glib-2.32.3" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.32" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2/2.30=", + "RDEPEND": "dev-libs/glib:2/2.30=" + }, + } + installed = { + "dev-libs/glib-1.2.10" : { + "EAPI": "4-slot-abi", + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2/2.30=", + "RDEPEND": "dev-libs/glib:2/2.30=" + }, + } + + world = ["dev-libs/glib:1", "dev-libs/dbus-glib"] + + test_cases = ( + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True}, + success = True, + mergelist = ["dev-libs/glib-2.32.3", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/glib-2.32.3"]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--usepkgonly": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.30.2"]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--usepkgonly": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ["dev-libs/glib-2.32.3", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/glib-2.32.3"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True}, + success = True, + mergelist = []), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3"]), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, binpkgs=binpkgs, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_abi_downgrade.py b/pym/portage/tests/resolver/test_slot_abi_downgrade.py new file mode 100644 index 00000000000..45a7555c21f --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_abi_downgrade.py @@ -0,0 +1,225 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SlotAbiDowngradeTestCase(TestCase): + + def __init__(self, *args, **kwargs): + super(SlotAbiDowngradeTestCase, self).__init__(*args, **kwargs) + + def testSubSlot(self): + ebuilds = { + "dev-libs/icu-4.8" : { + "EAPI": "4-slot-abi", + "SLOT": "0/48" + }, + "dev-libs/libxml2-2.7.8" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/icu:=", + "RDEPEND": "dev-libs/icu:=" + }, + } + binpkgs = { + "dev-libs/icu-49" : { + "EAPI": "4-slot-abi", + "SLOT": "0/49" + }, + "dev-libs/icu-4.8" : { + "EAPI": "4-slot-abi", + "SLOT": "0/48" + }, + "dev-libs/libxml2-2.7.8" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/icu:0/49=", + "RDEPEND": "dev-libs/icu:0/49=" + }, + } + installed = { + "dev-libs/icu-49" : { + "EAPI": "4-slot-abi", + "SLOT": "0/49" + }, + "dev-libs/libxml2-2.7.8" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/icu:0/49=", + "RDEPEND": "dev-libs/icu:0/49=" + }, + } + + world = ["dev-libs/libxml2"] + + test_cases = ( + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True}, + success = True, + mergelist = ["dev-libs/icu-4.8", "dev-libs/libxml2-2.7.8" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/icu-4.8"]), + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/icu-4.8", "dev-libs/libxml2-2.7.8" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/icu"], + options = {"--oneshot": True, "--usepkgonly": True}, + success = True, + mergelist = ["[binary]dev-libs/icu-49"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ["dev-libs/icu-4.8", "dev-libs/libxml2-2.7.8" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/icu-4.8"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/icu-4.8", "dev-libs/libxml2-2.7.8" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True}, + success = True, + mergelist = []), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, binpkgs=binpkgs, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() + + def testWholeSlotSubSlotMix(self): + ebuilds = { + "dev-libs/glib-1.2.10" : { + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2=", + "RDEPEND": "dev-libs/glib:2=" + }, + } + binpkgs = { + "dev-libs/glib-1.2.10" : { + "SLOT": "1" + }, + "dev-libs/glib-2.30.2" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.30" + }, + "dev-libs/glib-2.32.3" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.32" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2/2.32=", + "RDEPEND": "dev-libs/glib:2/2.32=" + }, + } + installed = { + "dev-libs/glib-1.2.10" : { + "EAPI": "4-slot-abi", + "SLOT": "1" + }, + "dev-libs/glib-2.32.3" : { + "EAPI": "4-slot-abi", + "SLOT": "2/2.32" + }, + "dev-libs/dbus-glib-0.98" : { + "EAPI": "4-slot-abi", + "DEPEND": "dev-libs/glib:2/2.32=", + "RDEPEND": "dev-libs/glib:2/2.32=" + }, + } + + world = ["dev-libs/glib:1", "dev-libs/dbus-glib"] + + test_cases = ( + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True}, + success = True, + mergelist = ["dev-libs/glib-2.30.2", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/glib-2.30.2"]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.30.2", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["dev-libs/glib"], + options = {"--oneshot": True, "--usepkgonly": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.32.3"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ["dev-libs/glib-2.30.2", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--ignore-built-slot-abi-deps": "y"}, + success = True, + mergelist = ["dev-libs/glib-2.30.2"]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkg": True}, + success = True, + mergelist = ["[binary]dev-libs/glib-2.30.2", "dev-libs/dbus-glib-0.98" ]), + + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True, "--usepkgonly": True}, + success = True, + mergelist = []), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, binpkgs=binpkgs, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_collisions.py b/pym/portage/tests/resolver/test_slot_collisions.py index 4867cea0508..95d68fe040a 100644 --- a/pym/portage/tests/resolver/test_slot_collisions.py +++ b/pym/portage/tests/resolver/test_slot_collisions.py @@ -45,6 +45,9 @@ class SlotCollisionTestCase(TestCase): "app-misc/C-1": { "DEPEND": "=app-misc/A-1[foo]", "EAPI": 2 }, "app-misc/E-1": { "RDEPEND": "dev-libs/E[foo?]", "IUSE": "foo", "EAPI": "2" }, "app-misc/F-1": { "RDEPEND": "=dev-libs/E-1", "IUSE": "foo", "EAPI": "2" }, + + "dev-lang/perl-5.12": {"SLOT": "0/5.12", "EAPI": "4-slot-abi"}, + "dev-lang/perl-5.16": {"SLOT": "0/5.16", "EAPI": "4-slot-abi"}, } installed = { "dev-libs/A-1": { "PDEPEND": "foo? ( dev-libs/B )", "IUSE": "foo", "USE": "foo" }, @@ -104,6 +107,15 @@ class SlotCollisionTestCase(TestCase): slot_collision_solutions = [] ), + # sub-slot + ResolverPlaygroundTestCase( + ["dev-lang/perl:0/5.12", "dev-lang/perl:0/5.16", "=dev-lang/perl-5.12*"], + success = False, + mergelist = ["dev-lang/perl-5.12", "dev-lang/perl-5.16"], + ignore_mergelist_order = True, + slot_collision_solutions = [] + ), + #Simple cases. ResolverPlaygroundTestCase( ["sci-libs/L", "sci-libs/M"], diff --git a/pym/portage/tests/runTests b/pym/portage/tests/runTests index 4c100870841..1c1008dfff2 100755 --- a/pym/portage/tests/runTests +++ b/pym/portage/tests/runTests @@ -1,18 +1,25 @@ #!/usr/bin/python -Wd # runTests.py -- Portage Unit Test Functionality -# Copyright 2006 Gentoo Foundation +# Copyright 2006-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os, sys import os.path as osp import grp +import platform import pwd import signal def debug_signal(signum, frame): import pdb pdb.set_trace() -signal.signal(signal.SIGUSR1, debug_signal) + +if platform.python_implementation() == 'Jython': + debug_signum = signal.SIGUSR2 # bug #424259 +else: + debug_signum = signal.SIGUSR1 + +signal.signal(debug_signum, debug_signal) # Pretend that the current user's uid/gid are the 'portage' uid/gid, # so things go smoothly regardless of the current user and global @@ -30,6 +37,9 @@ import portage # work the same regardless of global configuration file state/existence. portage._disable_legacy_globals() +if os.environ.get('NOCOLOR') in ('yes', 'true'): + portage.output.nocolor() + import portage.tests as tests from portage.const import PORTAGE_BIN_PATH path = os.environ.get("PATH", "").split(":") diff --git a/pym/portage/tests/update/__init__.py b/pym/portage/tests/update/__init__.py new file mode 100644 index 00000000000..418ad862bbc --- /dev/null +++ b/pym/portage/tests/update/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 diff --git a/pym/portage/tests/update/__test__ b/pym/portage/tests/update/__test__ new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/pym/portage/tests/update/__test__ diff --git a/pym/portage/tests/update/test_move_ent.py b/pym/portage/tests/update/test_move_ent.py new file mode 100644 index 00000000000..2504dee2b13 --- /dev/null +++ b/pym/portage/tests/update/test_move_ent.py @@ -0,0 +1,109 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +import portage +from portage import os +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.util import ensure_dirs +from portage._global_updates import _do_global_updates + +class MoveEntTestCase(TestCase): + + def testMoveEnt(self): + + ebuilds = { + + "dev-libs/A-2::dont_apply_updates" : { + "EAPI": "4", + "SLOT": "2", + }, + + } + + installed = { + + "dev-libs/A-1::test_repo" : { + "EAPI": "4", + }, + + "dev-libs/A-2::dont_apply_updates" : { + "EAPI": "4", + "SLOT": "2", + }, + + } + + binpkgs = { + + "dev-libs/A-1::test_repo" : { + "EAPI": "4", + }, + + "dev-libs/A-2::dont_apply_updates" : { + "EAPI": "4", + "SLOT": "2", + }, + + } + + updates = textwrap.dedent(""" + move dev-libs/A dev-libs/A-moved + """) + + playground = ResolverPlayground(binpkgs=binpkgs, + ebuilds=ebuilds, installed=installed) + + settings = playground.settings + trees = playground.trees + eroot = settings["EROOT"] + portdir = settings["PORTDIR"] + portdb = trees[eroot]["porttree"].dbapi + vardb = trees[eroot]["vartree"].dbapi + bindb = trees[eroot]["bintree"].dbapi + + updates_dir = os.path.join(portdir, "profiles", "updates") + + try: + ensure_dirs(updates_dir) + with open(os.path.join(updates_dir, "1Q-2010"), 'w') as f: + f.write(updates) + + # Create an empty updates directory, so that this + # repo doesn't inherit updates from the main repo. + ensure_dirs(os.path.join( + portdb.getRepositoryPath("dont_apply_updates"), + "profiles", "updates")) + + global_noiselimit = portage.util.noiselimit + portage.util.noiselimit = -2 + try: + _do_global_updates(trees, {}) + finally: + portage.util.noiselimit = global_noiselimit + + # Workaround for cache validation not working + # correctly when filesystem has timestamp precision + # of 1 second. + vardb._clear_cache() + + # A -> A-moved + self.assertRaises(KeyError, + vardb.aux_get, "dev-libs/A-1", ["EAPI"]) + vardb.aux_get("dev-libs/A-moved-1", ["EAPI"]) + self.assertRaises(KeyError, + bindb.aux_get, "dev-libs/A-1", ["EAPI"]) + bindb.aux_get("dev-libs/A-moved-1", ["EAPI"]) + + # dont_apply_updates + self.assertRaises(KeyError, + vardb.aux_get, "dev-libs/A-moved-2", ["EAPI"]) + vardb.aux_get("dev-libs/A-2", ["EAPI"]) + self.assertRaises(KeyError, + bindb.aux_get, "dev-libs/A-moved-2", ["EAPI"]) + bindb.aux_get("dev-libs/A-2", ["EAPI"]) + + finally: + playground.cleanup() diff --git a/pym/portage/tests/update/test_move_slot_ent.py b/pym/portage/tests/update/test_move_slot_ent.py new file mode 100644 index 00000000000..fcb0cc64c6a --- /dev/null +++ b/pym/portage/tests/update/test_move_slot_ent.py @@ -0,0 +1,154 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +import portage +from portage import os +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.util import ensure_dirs +from portage._global_updates import _do_global_updates + +class MoveSlotEntTestCase(TestCase): + + def testMoveSlotEnt(self): + + ebuilds = { + + "dev-libs/A-2::dont_apply_updates" : { + "EAPI": "4-slot-abi", + "SLOT": "0/2.30", + }, + + "dev-libs/B-2::dont_apply_updates" : { + "SLOT": "0", + }, + + "dev-libs/C-2.1::dont_apply_updates" : { + "EAPI": "4-slot-abi", + "SLOT": "0/2.1", + }, + + } + + installed = { + + "dev-libs/A-1::test_repo" : { + "EAPI": "4-slot-abi", + "SLOT": "0/2.30", + }, + + "dev-libs/B-1::test_repo" : { + "SLOT": "0", + }, + + "dev-libs/C-1::test_repo" : { + "EAPI": "4-slot-abi", + "SLOT": "0/1", + }, + + } + + binpkgs = { + + "dev-libs/A-1::test_repo" : { + "EAPI": "4-slot-abi", + "SLOT": "0/2.30", + }, + + "dev-libs/A-2::dont_apply_updates" : { + "EAPI": "4-slot-abi", + "SLOT": "0/2.30", + }, + + "dev-libs/B-1::test_repo" : { + "SLOT": "0", + }, + + "dev-libs/B-2::dont_apply_updates" : { + "SLOT": "0", + }, + + "dev-libs/C-1::test_repo" : { + "EAPI": "4-slot-abi", + "SLOT": "0/1", + }, + + "dev-libs/C-2.1::dont_apply_updates" : { + "EAPI": "4-slot-abi", + "SLOT": "0/2.1", + }, + + } + + updates = textwrap.dedent(""" + slotmove dev-libs/A 0 2 + slotmove dev-libs/B 0 1 + slotmove dev-libs/C 0 1 + """) + + playground = ResolverPlayground(binpkgs=binpkgs, + ebuilds=ebuilds, installed=installed) + + settings = playground.settings + trees = playground.trees + eroot = settings["EROOT"] + portdir = settings["PORTDIR"] + portdb = trees[eroot]["porttree"].dbapi + vardb = trees[eroot]["vartree"].dbapi + bindb = trees[eroot]["bintree"].dbapi + + updates_dir = os.path.join(portdir, "profiles", "updates") + + try: + ensure_dirs(updates_dir) + with open(os.path.join(updates_dir, "1Q-2010"), 'w') as f: + f.write(updates) + + # Create an empty updates directory, so that this + # repo doesn't inherit updates from the main repo. + ensure_dirs(os.path.join( + portdb.getRepositoryPath("dont_apply_updates"), + "profiles", "updates")) + + global_noiselimit = portage.util.noiselimit + portage.util.noiselimit = -2 + try: + _do_global_updates(trees, {}) + finally: + portage.util.noiselimit = global_noiselimit + + # Workaround for cache validation not working + # correctly when filesystem has timestamp precision + # of 1 second. + vardb._clear_cache() + + # 0/2.30 -> 2/2.30 + self.assertEqual("2/2.30", + vardb.aux_get("dev-libs/A-1", ["SLOT"])[0]) + self.assertEqual("2/2.30", + bindb.aux_get("dev-libs/A-1", ["SLOT"])[0]) + + # 0 -> 1 + self.assertEqual("1", + vardb.aux_get("dev-libs/B-1", ["SLOT"])[0]) + self.assertEqual("1", + bindb.aux_get("dev-libs/B-1", ["SLOT"])[0]) + + # 0/1 -> 1 (equivalent to 1/1) + self.assertEqual("1", + vardb.aux_get("dev-libs/C-1", ["SLOT"])[0]) + self.assertEqual("1", + bindb.aux_get("dev-libs/C-1", ["SLOT"])[0]) + + # dont_apply_updates + self.assertEqual("0/2.30", + bindb.aux_get("dev-libs/A-2", ["SLOT"])[0]) + self.assertEqual("0", + bindb.aux_get("dev-libs/B-2", ["SLOT"])[0]) + self.assertEqual("0/2.1", + bindb.aux_get("dev-libs/C-2.1", ["SLOT"])[0]) + + finally: + playground.cleanup() diff --git a/pym/portage/tests/update/test_update_dbentry.py b/pym/portage/tests/update/test_update_dbentry.py new file mode 100644 index 00000000000..e13cfed74bb --- /dev/null +++ b/pym/portage/tests/update/test_update_dbentry.py @@ -0,0 +1,184 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import re +import textwrap + +import portage +from portage import os +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.util import ensure_dirs +from portage._global_updates import _do_global_updates + +class UpdateDbentryTestCase(TestCase): + + def testUpdateDbentryTestCase(self): + + ebuilds = { + + "dev-libs/A-2::dont_apply_updates" : { + "RDEPEND" : "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4", + "SLOT": "2", + }, + + "dev-libs/B-2::dont_apply_updates" : { + "RDEPEND" : "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4", + "SLOT": "2", + }, + + } + + installed = { + + "dev-libs/A-1::test_repo" : { + "RDEPEND" : "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4", + }, + + "dev-libs/A-2::dont_apply_updates" : { + "RDEPEND" : "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4", + "SLOT": "2", + }, + + "dev-libs/B-1::test_repo" : { + "RDEPEND" : "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4-python", + }, + + "dev-libs/M-1::test_repo" : { + "EAPI": "4", + }, + + "dev-libs/N-1::test_repo" : { + "EAPI": "4", + }, + + "dev-libs/N-2::test_repo" : { + "EAPI": "4-python", + }, + + } + + binpkgs = { + + "dev-libs/A-1::test_repo" : { + "RDEPEND" : "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4", + }, + + "dev-libs/A-2::dont_apply_updates" : { + "RDEPEND" : "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4", + "SLOT": "2", + }, + + "dev-libs/B-1::test_repo" : { + "RDEPEND" : "dev-libs/M dev-libs/N dev-libs/P", + "EAPI": "4-python", + }, + + } + + world = ["dev-libs/M", "dev-libs/N"] + + updates = textwrap.dedent(""" + move dev-libs/M dev-libs/M-moved + move dev-libs/N dev-libs/N.moved + """) + + playground = ResolverPlayground(binpkgs=binpkgs, + ebuilds=ebuilds, installed=installed, world=world) + + settings = playground.settings + trees = playground.trees + eroot = settings["EROOT"] + portdir = settings["PORTDIR"] + portdb = trees[eroot]["porttree"].dbapi + vardb = trees[eroot]["vartree"].dbapi + bindb = trees[eroot]["bintree"].dbapi + setconfig = trees[eroot]["root_config"].setconfig + selected_set = setconfig.getSets()["selected"] + + updates_dir = os.path.join(portdir, "profiles", "updates") + + try: + ensure_dirs(updates_dir) + with open(os.path.join(updates_dir, "1Q-2010"), 'w') as f: + f.write(updates) + + # Create an empty updates directory, so that this + # repo doesn't inherit updates from the main repo. + ensure_dirs(os.path.join( + portdb.getRepositoryPath("dont_apply_updates"), + "profiles", "updates")) + + global_noiselimit = portage.util.noiselimit + portage.util.noiselimit = -2 + try: + _do_global_updates(trees, {}) + finally: + portage.util.noiselimit = global_noiselimit + + # Workaround for cache validation not working + # correctly when filesystem has timestamp precision + # of 1 second. + vardb._clear_cache() + + # M -> M-moved + old_pattern = re.compile(r"\bdev-libs/M(\s|$)") + rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0] + self.assertTrue(old_pattern.search(rdepend) is None) + self.assertTrue("dev-libs/M-moved" in rdepend) + rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0] + self.assertTrue(old_pattern.search(rdepend) is None) + self.assertTrue("dev-libs/M-moved" in rdepend) + rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0] + self.assertTrue(old_pattern.search(rdepend) is None) + self.assertTrue("dev-libs/M-moved" in rdepend) + rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0] + self.assertTrue(old_pattern.search(rdepend) is None) + self.assertTrue("dev-libs/M-moved" in rdepend) + + # EAPI 4-python N -> N.moved + rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0] + old_pattern = re.compile(r"\bdev-libs/N(\s|$)") + self.assertTrue(old_pattern.search(rdepend) is None) + self.assertTrue("dev-libs/N.moved" in rdepend) + rdepend = bindb.aux_get("dev-libs/B-1", ["RDEPEND"])[0] + self.assertTrue(old_pattern.search(rdepend) is None) + self.assertTrue("dev-libs/N.moved" in rdepend) + self.assertRaises(KeyError, + vardb.aux_get, "dev-libs/N-2", ["EAPI"]) + vardb.aux_get("dev-libs/N.moved-2", ["RDEPEND"])[0] + + # EAPI 4 does not allow dots in package names for N -> N.moved + rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0] + self.assertTrue("dev-libs/N" in rdepend) + self.assertTrue("dev-libs/N.moved" not in rdepend) + rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0] + self.assertTrue("dev-libs/N" in rdepend) + self.assertTrue("dev-libs/N.moved" not in rdepend) + vardb.aux_get("dev-libs/N-1", ["RDEPEND"])[0] + self.assertRaises(KeyError, + vardb.aux_get, "dev-libs/N.moved-1", ["EAPI"]) + + # dont_apply_updates + rdepend = vardb.aux_get("dev-libs/A-2", ["RDEPEND"])[0] + self.assertTrue("dev-libs/M" in rdepend) + self.assertTrue("dev-libs/M-moved" not in rdepend) + rdepend = bindb.aux_get("dev-libs/A-2", ["RDEPEND"])[0] + self.assertTrue("dev-libs/M" in rdepend) + self.assertTrue("dev-libs/M-moved" not in rdepend) + + selected_set.load() + self.assertTrue("dev-libs/M" not in selected_set) + self.assertTrue("dev-libs/M-moved" in selected_set) + self.assertTrue("dev-libs/N" not in selected_set) + self.assertTrue("dev-libs/N.moved" in selected_set) + + finally: + playground.cleanup() diff --git a/pym/portage/tests/util/test_digraph.py b/pym/portage/tests/util/test_digraph.py index b65c0b18e01..4e858cf88d2 100644 --- a/pym/portage/tests/util/test_digraph.py +++ b/pym/portage/tests/util/test_digraph.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -8,6 +8,41 @@ import portage.util class DigraphTest(TestCase): + def _assertBFSEqual(self, result, expected): + result_stack = list(result) + result_stack.reverse() + expected_stack = list(reversed(expected)) + result_compared = [] + expected_compared = [] + while result_stack: + if not expected_stack: + result_compared.append(result_stack.pop()) + self.assertEqual(result_compared, expected_compared) + expected_set = expected_stack.pop() + if not isinstance(expected_set, list): + expected_set = [expected_set] + expected_set = set(expected_set) + while expected_set: + if not result_stack: + expected_compared.extend(expected_set) + self.assertEqual(result_compared, expected_compared) + obj = result_stack.pop() + try: + expected_set.remove(obj) + except KeyError: + expected_compared.extend(expected_set) + result_compared.append(obj) + self.assertEqual(result_compared, expected_compared) + else: + expected_compared.append(obj) + result_compared.append(obj) + if expected_stack: + expected_set = expected_stack.pop() + if not isinstance(expected_set, list): + expected_set = [expected_set] + expected_compared.extend(expected_set) + self.assertEqual(result_compared, expected_compared) + def testBackwardCompatibility(self): g = digraph() f = g.copy() @@ -71,7 +106,7 @@ class DigraphTest(TestCase): self.assertEqual(x.parent_nodes("A", ignore_priority=-2), ["B"]) self.assertEqual(x.parent_nodes("A", ignore_priority=-1), []) self.assertEqual(x.hasallzeros(), False) - self.assertEqual(list(x.bfs("A")), [(None, "A"), ("A", "D"), ("D", "C"), ("C", "B")]) + self._assertBFSEqual(x.bfs("A"), [(None, "A"), ("A", "D"), ("D", "C"), ("C", "B")]) self.assertEqual(x.shortest_path("A", "D"), ["A", "D"]) self.assertEqual(x.shortest_path("D", "A"), ["D", "C", "B", "A"]) self.assertEqual(x.shortest_path("A", "D", ignore_priority=2), None) @@ -115,7 +150,7 @@ class DigraphTest(TestCase): self.assertEqual(x.parent_nodes("B", ignore_priority=-2), ["A"]) self.assertEqual(x.parent_nodes("B", ignore_priority=-1), []) self.assertEqual(x.hasallzeros(), False) - self.assertEqual(list(x.bfs("A")), [(None, "A"), ("A", "C"), ("A", "B"), ("C", "E"), ("C", "D")]) + self._assertBFSEqual(x.bfs("A"), [(None, "A"), [("A", "C"), ("A", "B")], [("C", "E"), ("C", "D")]]) self.assertEqual(x.shortest_path("A", "D"), ["A", "C", "D"]) self.assertEqual(x.shortest_path("D", "A"), None) self.assertEqual(x.shortest_path("A", "D", ignore_priority=2), None) @@ -158,15 +193,16 @@ class DigraphTest(TestCase): self.assertEqual(x.parent_nodes("A", ignore_priority=0), ["C"]) self.assertEqual(x.parent_nodes("A", ignore_priority=1), []) self.assertEqual(x.hasallzeros(), False) - self.assertEqual(list(x.bfs("A")), [(None, "A"), ("A", "C"), ("A", "B")]) + self._assertBFSEqual(x.bfs("A"), [(None, "A"), [("A", "C"), ("A", "B")]]) self.assertEqual(x.shortest_path("A", "C"), ["A", "C"]) self.assertEqual(x.shortest_path("C", "A"), ["C", "A"]) self.assertEqual(x.shortest_path("A", "C", ignore_priority=0), ["A", "B", "C"]) self.assertEqual(x.shortest_path("C", "A", ignore_priority=0), ["C", "A"]) - cycles = set(tuple(y) for y in x.get_cycles()) - self.assertEqual(cycles, set([("C", "A"), ("A", "B"), ("A", "C")])) + cycles = set(frozenset(y) for y in x.get_cycles()) + self.assertEqual(cycles, set([frozenset(["A", "B"]), frozenset(["A", "C"]), frozenset(["B", "C"])])) x.remove_edge("A", "B") - self.assertEqual(x.get_cycles(), [["C", "A"], ["A", "C"], ["C", "B"]]) + cycles = set(frozenset(y) for y in x.get_cycles()) + self.assertEqual(cycles, set([frozenset(["A", "C"]), frozenset(["C", "B"])])) x.difference_update(["C"]) self.assertEqual(x.all_nodes(), ["A", "B"]) portage.util.noiselimit = -2 diff --git a/pym/portage/tests/util/test_stackLists.py b/pym/portage/tests/util/test_stackLists.py index 8d01ea5ac8e..e5247725576 100644 --- a/pym/portage/tests/util/test_stackLists.py +++ b/pym/portage/tests/util/test_stackLists.py @@ -1,5 +1,5 @@ # test_stackLists.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation +# Copyright 2006-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -16,4 +16,4 @@ class StackListsTestCase(TestCase): for test in tests: result = stack_lists( test[0], test[2] ) - self.assertEqual( result , test[1] ) + self.assertEqual( set(result) , set(test[1]) ) diff --git a/pym/portage/update.py b/pym/portage/update.py index 34e466366e3..121e957201c 100644 --- a/pym/portage/update.py +++ b/pym/portage/update.py @@ -18,23 +18,32 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:ConfigProtect,new_protect_filename,' + \ 'normalize_path,write_atomic,writemsg', 'portage.util.listdir:_ignorecvs_dirs', - 'portage.versions:ververify' + 'portage.versions:catsplit,ververify' ) from portage.const import USER_CONFIG_PATH +from portage.dep import _get_slot_re +from portage.eapi import _get_eapi_attrs from portage.exception import DirectoryNotFound, InvalidAtom, PortageException from portage.localization import _ if sys.hexversion >= 0x3000000: long = int + _unicode = str +else: + _unicode = unicode ignored_dbentries = ("CONTENTS", "environment.bz2") -def update_dbentry(update_cmd, mycontent): +def update_dbentry(update_cmd, mycontent, eapi=None): + if update_cmd[0] == "move": - old_value = str(update_cmd[1]) - if old_value in mycontent: - new_value = str(update_cmd[2]) + old_value = _unicode(update_cmd[1]) + new_value = _unicode(update_cmd[2]) + + # Use isvalidatom() to check if this move is valid for the + # EAPI (characters allowed in package names may vary). + if old_value in mycontent and isvalidatom(new_value, eapi=eapi): old_value = re.escape(old_value); mycontent = re.sub(old_value+"(:|$|\\s)", new_value+"\\1", mycontent) def myreplace(matchobj): @@ -56,7 +65,7 @@ def update_dbentry(update_cmd, mycontent): mycontent = re.sub(old_value+"($|\\s)", new_value+"\\1", mycontent) return mycontent -def update_dbentries(update_iter, mydata): +def update_dbentries(update_iter, mydata, eapi=None): """Performs update commands and returns a dict containing only the updated items.""" updated_items = {} @@ -70,7 +79,7 @@ def update_dbentries(update_iter, mydata): is_encoded = mycontent is not orig_content orig_content = mycontent for update_cmd in update_iter: - mycontent = update_dbentry(update_cmd, mycontent) + mycontent = update_dbentry(update_cmd, mycontent, eapi=eapi) if mycontent != orig_content: if is_encoded: mycontent = _unicode_encode(mycontent, @@ -79,7 +88,7 @@ def update_dbentries(update_iter, mydata): updated_items[k] = mycontent return updated_items -def fixdbentries(update_iter, dbdir): +def fixdbentries(update_iter, dbdir, eapi=None): """Performs update commands which result in search and replace operations for each of the files in dbdir (excluding CONTENTS and environment.bz2). Returns True when actual modifications are necessary and False otherwise.""" @@ -91,7 +100,7 @@ def fixdbentries(update_iter, dbdir): mode='r', encoding=_encodings['repo.content'], errors='replace') as f: mydata[myfile] = f.read() - updated_items = update_dbentries(update_iter, mydata) + updated_items = update_dbentries(update_iter, mydata, eapi=eapi) for myfile, mycontent in updated_items.items(): file_path = os.path.join(dbdir, myfile) write_atomic(file_path, mycontent, encoding=_encodings['repo.content']) @@ -143,6 +152,8 @@ def grab_updates(updpath, prev_mtimes=None): def parse_updates(mycontent): """Valid updates are returned as a list of split update commands.""" + eapi_attrs = _get_eapi_attrs(None) + slot_re = _get_slot_re(eapi_attrs) myupd = [] errors = [] mylines = mycontent.splitlines() @@ -194,6 +205,21 @@ def parse_updates(mycontent): errors.append(_("ERROR: Malformed update entry '%s'") % myline) continue + invalid_slot = False + for slot in (origslot, newslot): + m = slot_re.match(slot) + if m is None: + invalid_slot = True + break + if "/" in slot: + # EAPI 4-slot-abi style SLOT is currently not supported. + invalid_slot = True + break + + if invalid_slot: + errors.append(_("ERROR: Malformed update entry '%s'") % myline) + continue + # The list of valid updates is filtered by continue statements above. myupd.append(mysplit) return myupd, errors diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index d6ac46c83ee..4645be52f21 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2004-2011 Gentoo Foundation +# Copyright 2004-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['apply_permissions', 'apply_recursive_permissions', @@ -41,7 +41,7 @@ from portage import _os_merge from portage import _unicode_encode from portage import _unicode_decode from portage.exception import InvalidAtom, PortageException, FileNotFound, \ - OperationNotPermitted, PermissionDenied, ReadOnlyFileSystem + OperationNotPermitted, ParseError, PermissionDenied, ReadOnlyFileSystem from portage.localization import _ from portage.proxy.objectproxy import ObjectProxy from portage.cache.mappings import UserDict @@ -447,7 +447,7 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0, allow_wildcard=Fals try: pkg = Atom(pkg, allow_wildcard=allow_wildcard, allow_repo=allow_repo, eapi=eapi) except InvalidAtom as e: - writemsg(_("--- Invalid atom in %s: %s\n") % (myfilename, e), + writemsg(_("--- Invalid atom in %s: %s\n") % (source_file, e), noiselevel=-1) else: if pkg_orig == str(pkg): @@ -469,7 +469,15 @@ def grablines(myfilename, recursive=0, remember_source_file=False): if recursive and os.path.isdir(myfilename): if os.path.basename(myfilename) in _ignorecvs_dirs: return mylines - dirlist = os.listdir(myfilename) + try: + dirlist = os.listdir(myfilename) + except OSError as e: + if e.errno == PermissionDenied.errno: + raise PermissionDenied(myfilename) + elif e.errno in (errno.ENOENT, errno.ESTALE): + return mylines + else: + raise dirlist.sort() for f in dirlist: if not f.startswith(".") and not f.endswith("~"): @@ -488,7 +496,10 @@ def grablines(myfilename, recursive=0, remember_source_file=False): except IOError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(myfilename) - pass + elif e.errno in (errno.ENOENT, errno.ESTALE): + pass + else: + raise return mylines def writedict(mydict,myfilename,writekey=True): @@ -573,6 +584,7 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): writemsg(("!!! " + _("Please use dos2unix to convert line endings " + \ "in config file: '%s'") + "\n") % mycfg, noiselevel=-1) + lex = None try: if tolerant: shlex_class = _tolerant_shlex @@ -596,43 +608,38 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): break; equ=lex.get_token() if (equ==''): - #unexpected end of file - #lex.error_leader(self.filename,lex.lineno) + msg = lex.error_leader() + _("Unexpected EOF") if not tolerant: - writemsg(_("!!! Unexpected end of config file: variable %s\n") % key, - noiselevel=-1) - raise Exception(_("ParseError: Unexpected EOF: %s: on/before line %s") % (mycfg, lex.lineno)) + raise ParseError(msg) else: + writemsg("%s\n" % msg, noiselevel=-1) return mykeys elif (equ!='='): - #invalid token - #lex.error_leader(self.filename,lex.lineno) + msg = lex.error_leader() + \ + _("Invalid token '%s' (not '=')") % (equ,) if not tolerant: - raise Exception(_("ParseError: Invalid token " - "'%s' (not '='): %s: line %s") % \ - (equ, mycfg, lex.lineno)) + raise ParseError(msg) else: + writemsg("%s\n" % msg, noiselevel=-1) return mykeys val=lex.get_token() if val is None: - #unexpected end of file - #lex.error_leader(self.filename,lex.lineno) + msg = lex.error_leader() + \ + _("Unexpected end of config file: variable '%s'") % (key,) if not tolerant: - writemsg(_("!!! Unexpected end of config file: variable %s\n") % key, - noiselevel=-1) - raise portage.exception.CorruptionError(_("ParseError: Unexpected EOF: %s: line %s") % (mycfg, lex.lineno)) + raise ParseError(msg) else: + writemsg("%s\n" % msg, noiselevel=-1) return mykeys key = _unicode_decode(key) val = _unicode_decode(val) if _invalid_var_name_re.search(key) is not None: + msg = lex.error_leader() + \ + _("Invalid variable name '%s'") % (key,) if not tolerant: - raise Exception(_( - "ParseError: Invalid variable name '%s': line %s") % \ - (key, lex.lineno - 1)) - writemsg(_("!!! Invalid variable name '%s': line %s in %s\n") \ - % (key, lex.lineno - 1, mycfg), noiselevel=-1) + raise ParseError(msg) + writemsg("%s\n" % msg, noiselevel=-1) continue if expand: @@ -644,7 +651,12 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): except SystemExit as e: raise except Exception as e: - raise portage.exception.ParseError(str(e)+" in "+mycfg) + if isinstance(e, ParseError) or lex is None: + raise + msg = _unicode_decode("%s%s") % (lex.error_leader(), e) + writemsg("%s\n" % msg, noiselevel=-1) + raise + return mykeys _varexpand_word_chars = frozenset(string.ascii_letters + string.digits + "_") diff --git a/pym/portage/util/_desktop_entry.py b/pym/portage/util/_desktop_entry.py new file mode 100644 index 00000000000..79017801376 --- /dev/null +++ b/pym/portage/util/_desktop_entry.py @@ -0,0 +1,75 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import io +import subprocess +import sys + +try: + from configparser import Error as ConfigParserError, RawConfigParser +except ImportError: + from ConfigParser import Error as ConfigParserError, RawConfigParser + +from portage import _encodings, _unicode_encode, _unicode_decode + +def parse_desktop_entry(path): + """ + Parse the given file with RawConfigParser and return the + result. This may raise an IOError from io.open(), or a + ParsingError from RawConfigParser. + """ + parser = RawConfigParser() + + # use read_file/readfp in order to control decoding of unicode + try: + # Python >=3.2 + read_file = parser.read_file + except AttributeError: + read_file = parser.readfp + + with io.open(_unicode_encode(path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + read_file(f) + + return parser + +_ignored_service_errors = ( + 'error: required key "Name" in group "Desktop Entry" is not present', + 'error: key "Actions" is present in group "Desktop Entry", but the type is "Service" while this key is only valid for type "Application"', + 'error: key "MimeType" is present in group "Desktop Entry", but the type is "Service" while this key is only valid for type "Application"', +) + +def validate_desktop_entry(path): + args = ["desktop-file-validate", path] + if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: + # Python 3.1 does not support bytes in Popen args. + args = [_unicode_encode(x, errors='strict') for x in args] + proc = subprocess.Popen(args, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output_lines = _unicode_decode(proc.communicate()[0]).splitlines() + proc.wait() + + if output_lines: + try: + desktop_entry = parse_desktop_entry(path) + except ConfigParserError: + pass + else: + if desktop_entry.has_section("Desktop Entry"): + try: + entry_type = desktop_entry.get("Desktop Entry", "Type") + except ConfigParserError: + pass + else: + if entry_type == "Service": + # Filter false errors for Type=Service (bug #414125). + filtered_output = [] + for line in output_lines: + if line[len(path)+2:] in _ignored_service_errors: + continue + filtered_output.append(line) + output_lines = filtered_output + + return output_lines diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index ef20ce40106..bbbce5261c0 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -35,7 +35,15 @@ class EventLoop(object): __slots__ = ("args", "function", "calling", "interval", "source_id", "timestamp") - def __init__(self): + def __init__(self, main=True): + """ + @param main: If True then this is a singleton instance for use + in the main thread, otherwise it is a local instance which + can safely be use in a non-main thread (default is True, so + that global_event_loop does not need constructor arguments) + @type main: bool + """ + self._use_signal = main self._poll_event_queue = [] self._poll_event_handlers = {} self._poll_event_handler_ids = {} @@ -198,20 +206,22 @@ class EventLoop(object): self._child_handlers[source_id] = self._child_callback_class( callback=callback, data=data, pid=pid, source_id=source_id) - if self._sigchld_read is None: - self._sigchld_read, self._sigchld_write = os.pipe() - fcntl.fcntl(self._sigchld_read, fcntl.F_SETFL, - fcntl.fcntl(self._sigchld_read, fcntl.F_GETFL) | os.O_NONBLOCK) - - # The IO watch is dynamically registered and unregistered as - # needed, since we don't want to consider it as a valid source - # of events when there are no child listeners. It's important - # to distinguish when there are no valid sources of IO events, - # in order to avoid an endless poll call if there's no timeout. - if self._sigchld_src_id is None: - self._sigchld_src_id = self.io_add_watch( - self._sigchld_read, self.IO_IN, self._sigchld_io_cb) - signal.signal(signal.SIGCHLD, self._sigchld_sig_cb) + if self._use_signal: + if self._sigchld_read is None: + self._sigchld_read, self._sigchld_write = os.pipe() + fcntl.fcntl(self._sigchld_read, fcntl.F_SETFL, + fcntl.fcntl(self._sigchld_read, + fcntl.F_GETFL) | os.O_NONBLOCK) + + # The IO watch is dynamically registered and unregistered as + # needed, since we don't want to consider it as a valid source + # of events when there are no child listeners. It's important + # to distinguish when there are no valid sources of IO events, + # in order to avoid an endless poll call if there's no timeout. + if self._sigchld_src_id is None: + self._sigchld_src_id = self.io_add_watch( + self._sigchld_read, self.IO_IN, self._sigchld_io_cb) + signal.signal(signal.SIGCHLD, self._sigchld_sig_cb) # poll now, in case the SIGCHLD has already arrived self._poll_child_processes() @@ -318,10 +328,15 @@ class EventLoop(object): def _run_timeouts(self): + calls = 0 + if not self._use_signal: + if self._poll_child_processes(): + calls += 1 + self._run_idle_callbacks() if not self._timeout_handlers: - return False + return bool(calls) ready_timeouts = [] current_time = time.time() @@ -334,7 +349,6 @@ class EventLoop(object): # Iterate of our local list, since self._timeout_handlers can be # modified during the exection of these callbacks. - calls = 0 for x in ready_timeouts: if x.source_id not in self._timeout_handlers: # it got cancelled while executing another timeout @@ -387,7 +401,7 @@ class EventLoop(object): """ x = self._child_handlers.pop(reg_id, None) if x is not None: - if not self._child_handlers: + if not self._child_handlers and self._use_signal: signal.signal(signal.SIGCHLD, signal.SIG_DFL) self.source_remove(self._sigchld_src_id) self._sigchld_src_id = None diff --git a/pym/portage/util/_urlopen.py b/pym/portage/util/_urlopen.py new file mode 100644 index 00000000000..307624bc474 --- /dev/null +++ b/pym/portage/util/_urlopen.py @@ -0,0 +1,42 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import sys + +try: + from urllib.request import urlopen as _urlopen + import urllib.parse as urllib_parse + import urllib.request as urllib_request + from urllib.parse import splituser as urllib_parse_splituser +except ImportError: + from urllib import urlopen as _urlopen + import urlparse as urllib_parse + import urllib2 as urllib_request + from urllib import splituser as urllib_parse_splituser + +def urlopen(url): + try: + return _urlopen(url) + except SystemExit: + raise + except Exception: + if sys.hexversion < 0x3000000: + raise + parse_result = urllib_parse.urlparse(url) + if parse_result.scheme not in ("http", "https") or \ + not parse_result.username: + raise + + return _new_urlopen(url) + +def _new_urlopen(url): + # This is experimental code for bug #413983. + parse_result = urllib_parse.urlparse(url) + netloc = urllib_parse_splituser(parse_result.netloc)[1] + url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment)) + password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm() + if parse_result.username is not None: + password_manager.add_password(None, url, parse_result.username, parse_result.password) + auth_handler = urllib_request.HTTPBasicAuthHandler(password_manager) + opener = urllib_request.build_opener(auth_handler) + return opener.open(url) diff --git a/pym/portage/util/digraph.py b/pym/portage/util/digraph.py index 1bbe10f615e..f3ae658c946 100644 --- a/pym/portage/util/digraph.py +++ b/pym/portage/util/digraph.py @@ -317,16 +317,23 @@ class digraph(object): """ all_cycles = [] for node in self.nodes: + # If we have multiple paths of the same length, we have to + # return them all, so that we always get the same results + # even with PYTHONHASHSEED="random" enabled. shortest_path = None + candidates = [] for child in self.child_nodes(node, ignore_priority): path = self.shortest_path(child, node, ignore_priority) if path is None: continue - if not shortest_path or len(shortest_path) > len(path): + if not shortest_path or len(shortest_path) >= len(path): shortest_path = path - if shortest_path: - if not max_length or len(shortest_path) <= max_length: - all_cycles.append(shortest_path) + candidates.append(path) + if shortest_path and \ + (not max_length or len(shortest_path) <= max_length): + for path in candidates: + if len(path) == len(shortest_path): + all_cycles.append(path) return all_cycles # Backward compatibility diff --git a/pym/portage/util/movefile.py b/pym/portage/util/movefile.py index 5ffd16bf03a..10577b5659d 100644 --- a/pym/portage/util/movefile.py +++ b/pym/portage/util/movefile.py @@ -141,9 +141,9 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if stat.S_ISLNK(sstat[stat.ST_MODE]): try: target=os.readlink(src) - if mysettings and mysettings["D"]: - if target.find(mysettings["D"])==0: - target=target[len(mysettings["D"]):] + if mysettings and "D" in mysettings and \ + target.startswith(mysettings["D"]): + target = target[len(mysettings["D"])-1:] if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): os.unlink(dest) try: diff --git a/pym/portage/versions.py b/pym/portage/versions.py index f8691d1ce22..27947532bf5 100644 --- a/pym/portage/versions.py +++ b/pym/portage/versions.py @@ -1,5 +1,5 @@ # versions.py -- core Portage functionality -# Copyright 1998-2010 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -9,14 +9,27 @@ __all__ = [ ] import re +import sys import warnings +if sys.hexversion < 0x3000000: + _unicode = unicode +else: + _unicode = str + import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util:cmp_sort_key' + 'portage.dep:_get_slot_re', + 'portage.repository.config:_gen_valid_repo', + 'portage.util:cmp_sort_key', ) +from portage import _unicode_decode +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidData from portage.localization import _ +_unknown_repo = "__unknown__" + # \w is [a-zA-Z0-9_] # 2.1.1 A category name may contain any of the characters [A-Za-z0-9+_.-]. @@ -26,21 +39,51 @@ _cat = r'[\w+][\w+.-]*' # 2.1.2 A package name may contain any of the characters [A-Za-z0-9+_-]. # It must not begin with a hyphen, # and must not end in a hyphen followed by one or more digits. -_pkg = r'[\w+][\w+-]*?' +_pkg = { + "dots_disallowed_in_PN": r'[\w+][\w+-]*?', + "dots_allowed_in_PN": r'[\w+][\w+.-]*?', +} _v = r'(cvs\.)?(\d+)((\.\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\d*)*)' _rev = r'\d+' _vr = _v + '(-r(' + _rev + '))?' -_cp = '(' + _cat + '/' + _pkg + '(-' + _vr + ')?)' -_cpv = '(' + _cp + '-' + _vr + ')' -_pv = '(?P<pn>' + _pkg + '(?P<pn_inval>-' + _vr + ')?)' + '-(?P<ver>' + _v + ')(-r(?P<rev>' + _rev + '))?' +_cp = { + "dots_disallowed_in_PN": '(' + _cat + '/' + _pkg['dots_disallowed_in_PN'] + '(-' + _vr + ')?)', + "dots_allowed_in_PN": '(' + _cat + '/' + _pkg['dots_allowed_in_PN'] + '(-' + _vr + ')?)', +} +_cpv = { + "dots_disallowed_in_PN": '(' + _cp['dots_disallowed_in_PN'] + '-' + _vr + ')', + "dots_allowed_in_PN": '(' + _cp['dots_allowed_in_PN'] + '-' + _vr + ')', +} +_pv = { + "dots_disallowed_in_PN": '(?P<pn>' + _pkg['dots_disallowed_in_PN'] + '(?P<pn_inval>-' + _vr + ')?)' + '-(?P<ver>' + _v + ')(-r(?P<rev>' + _rev + '))?', + "dots_allowed_in_PN": '(?P<pn>' + _pkg['dots_allowed_in_PN'] + '(?P<pn_inval>-' + _vr + ')?)' + '-(?P<ver>' + _v + ')(-r(?P<rev>' + _rev + '))?', +} ver_regexp = re.compile("^" + _vr + "$") suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$") suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1} endversion_keys = ["pre", "p", "alpha", "beta", "rc"] +_pv_re_cache = {} + +def _get_pv_re(eapi_attrs): + cache_key = eapi_attrs.dots_in_PN + pv_re = _pv_re_cache.get(cache_key) + if pv_re is not None: + return pv_re + + if eapi_attrs.dots_in_PN: + pv_re = _pv['dots_allowed_in_PN'] + else: + pv_re = _pv['dots_disallowed_in_PN'] + + pv_re = re.compile('^' + pv_re + '$', re.VERBOSE) + + _pv_re_cache[cache_key] = pv_re + return pv_re + def ververify(myver, silent=1): if ver_regexp.match(myver): return 1 @@ -49,7 +92,6 @@ def ververify(myver, silent=1): print(_("!!! syntax error in version: %s") % myver) return 0 -vercmp_cache = {} def vercmp(ver1, ver2, silent=1): """ Compare two versions @@ -76,11 +118,7 @@ def vercmp(ver1, ver2, silent=1): if ver1 == ver2: return 0 - mykey=ver1+":"+ver2 - try: - return vercmp_cache[mykey] - except KeyError: - pass + match1 = ver_regexp.match(ver1) match2 = ver_regexp.match(ver2) @@ -96,10 +134,8 @@ def vercmp(ver1, ver2, silent=1): # shortcut for cvs ebuilds (new style) if match1.group(1) and not match2.group(1): - vercmp_cache[mykey] = 1 return 1 elif match2.group(1) and not match1.group(1): - vercmp_cache[mykey] = -1 return -1 # building lists of the version parts before the suffix @@ -153,16 +189,13 @@ def vercmp(ver1, ver2, silent=1): for i in range(0, max(len(list1), len(list2))): if len(list1) <= i: - vercmp_cache[mykey] = -1 return -1 elif len(list2) <= i: - vercmp_cache[mykey] = 1 return 1 elif list1[i] != list2[i]: a = list1[i] b = list2[i] rval = (a > b) - (a < b) - vercmp_cache[mykey] = rval return rval # main version is equal, so now compare the _suffix part @@ -183,7 +216,6 @@ def vercmp(ver1, ver2, silent=1): a = suffix_value[s1[0]] b = suffix_value[s2[0]] rval = (a > b) - (a < b) - vercmp_cache[mykey] = rval return rval if s1[1] != s2[1]: # it's possible that the s(1|2)[1] == '' @@ -198,7 +230,6 @@ def vercmp(ver1, ver2, silent=1): r2 = 0 rval = (r1 > r2) - (r1 < r2) if rval: - vercmp_cache[mykey] = rval return rval # the suffix part is equal to, so finally check the revision @@ -211,7 +242,6 @@ def vercmp(ver1, ver2, silent=1): else: r2 = 0 rval = (r1 > r2) - (r1 < r2) - vercmp_cache[mykey] = rval return rval def pkgcmp(pkg1, pkg2): @@ -240,16 +270,14 @@ def pkgcmp(pkg1, pkg2): return None return vercmp("-".join(pkg1[1:]), "-".join(pkg2[1:])) -_pv_re = re.compile('^' + _pv + '$', re.VERBOSE) - -def _pkgsplit(mypkg): +def _pkgsplit(mypkg, eapi=None): """ @param mypkg: pv @return: 1. None if input is invalid. 2. (pn, ver, rev) if input is pv """ - m = _pv_re.match(mypkg) + m = _get_pv_re(_get_eapi_attrs(eapi)).match(mypkg) if m is None: return None @@ -266,8 +294,8 @@ def _pkgsplit(mypkg): _cat_re = re.compile('^%s$' % _cat) _missing_cat = 'null' -catcache={} -def catpkgsplit(mydata,silent=1): + +def catpkgsplit(mydata, silent=1, eapi=None): """ Takes a Category/Package-Version-Rev and returns a list of each. @@ -281,28 +309,85 @@ def catpkgsplit(mydata,silent=1): 2. If cat is not specificed in mydata, cat will be "null" 3. if rev does not exist it will be '-r0' """ - try: - return catcache[mydata] - except KeyError: + return mydata.cpv_split + except AttributeError: pass mysplit = mydata.split('/', 1) p_split=None if len(mysplit)==1: cat = _missing_cat - p_split = _pkgsplit(mydata) + p_split = _pkgsplit(mydata, eapi=eapi) elif len(mysplit)==2: cat = mysplit[0] if _cat_re.match(cat) is not None: - p_split = _pkgsplit(mysplit[1]) + p_split = _pkgsplit(mysplit[1], eapi=eapi) if not p_split: - catcache[mydata]=None return None retval = (cat, p_split[0], p_split[1], p_split[2]) - catcache[mydata]=retval return retval -def pkgsplit(mypkg, silent=1): +class _pkg_str(_unicode): + """ + This class represents a cpv. It inherits from str (unicode in python2) and + has attributes that cache results for use by functions like catpkgsplit and + cpv_getkey which are called frequently (especially in match_from_list). + Instances are typically created in dbapi.cp_list() or the Atom contructor, + and propagate from there. Generally, code that pickles these objects will + manually convert them to a plain unicode object first. + """ + + def __new__(cls, cpv, slot=None, repo=None, eapi=None): + return _unicode.__new__(cls, cpv) + + def __init__(self, cpv, slot=None, repo=None, eapi=None): + if not isinstance(cpv, _unicode): + # Avoid TypeError from _unicode.__init__ with PyPy. + cpv = _unicode_decode(cpv) + _unicode.__init__(cpv) + if eapi is not None: + self.__dict__['eapi'] = eapi + self.__dict__['cpv_split'] = catpkgsplit(cpv, eapi=eapi) + if self.cpv_split is None: + raise InvalidData(cpv) + self.__dict__['cp'] = self.cpv_split[0] + '/' + self.cpv_split[1] + if self.cpv_split[-1] == "r0" and cpv[-3:] != "-r0": + self.__dict__['version'] = "-".join(self.cpv_split[2:-1]) + else: + self.__dict__['version'] = "-".join(self.cpv_split[2:]) + # for match_from_list introspection + self.__dict__['cpv'] = self + if slot is not None: + eapi_attrs = _get_eapi_attrs(eapi) + slot_match = _get_slot_re(eapi_attrs).match(slot) + if slot_match is None: + # Avoid an InvalidAtom exception when creating SLOT atoms + self.__dict__['slot'] = '0' + self.__dict__['slot_abi'] = '0' + self.__dict__['slot_invalid'] = slot + else: + if eapi_attrs.slot_abi: + slot_split = slot.split("/") + self.__dict__['slot'] = slot_split[0] + if len(slot_split) > 1: + self.__dict__['slot_abi'] = slot_split[1] + else: + self.__dict__['slot_abi'] = slot_split[0] + else: + self.__dict__['slot'] = slot + self.__dict__['slot_abi'] = slot + + if repo is not None: + repo = _gen_valid_repo(repo) + if not repo: + repo = _unknown_repo + self.__dict__['repo'] = repo + + def __setattr__(self, name, value): + raise AttributeError("_pkg_str instances are immutable", + self.__class__, name, value) + +def pkgsplit(mypkg, silent=1, eapi=None): """ @param mypkg: either a pv or cpv @return: @@ -310,7 +395,7 @@ def pkgsplit(mypkg, silent=1): 2. (pn, ver, rev) if input is pv 3. (cp, ver, rev) if input is a cpv """ - catpsplit = catpkgsplit(mypkg) + catpsplit = catpkgsplit(mypkg, eapi=eapi) if catpsplit is None: return None cat, pn, ver, rev = catpsplit @@ -319,9 +404,13 @@ def pkgsplit(mypkg, silent=1): else: return (cat + '/' + pn, ver, rev) -def cpv_getkey(mycpv): +def cpv_getkey(mycpv, eapi=None): """Calls catpkgsplit on a cpv and returns only the cp.""" - mysplit = catpkgsplit(mycpv) + try: + return mycpv.cp + except AttributeError: + pass + mysplit = catpkgsplit(mycpv, eapi=eapi) if mysplit is not None: return mysplit[0] + '/' + mysplit[1] @@ -330,7 +419,7 @@ def cpv_getkey(mycpv): DeprecationWarning, stacklevel=2) myslash = mycpv.split("/", 1) - mysplit = _pkgsplit(myslash[-1]) + mysplit = _pkgsplit(myslash[-1], eapi=eapi) if mysplit is None: return None mylen = len(myslash) @@ -339,14 +428,18 @@ def cpv_getkey(mycpv): else: return mysplit[0] -def cpv_getversion(mycpv): +def cpv_getversion(mycpv, eapi=None): """Returns the v (including revision) from an cpv.""" - cp = cpv_getkey(mycpv) + try: + return mycpv.version + except AttributeError: + pass + cp = cpv_getkey(mycpv, eapi=eapi) if cp is None: return None return mycpv[len(cp+"-"):] -def cpv_sort_key(): +def cpv_sort_key(eapi=None): """ Create an object for sorting cpvs, to be used as the 'key' parameter in places like list.sort() or sorted(). This calls catpkgsplit() once for @@ -365,39 +458,55 @@ def cpv_sort_key(): split1 = split_cache.get(cpv1, False) if split1 is False: - split1 = catpkgsplit(cpv1) - if split1 is not None: - split1 = (split1[:2], '-'.join(split1[2:])) + split1 = None + try: + split1 = cpv1.cpv + except AttributeError: + try: + split1 = _pkg_str(cpv1, eapi=eapi) + except InvalidData: + pass split_cache[cpv1] = split1 split2 = split_cache.get(cpv2, False) if split2 is False: - split2 = catpkgsplit(cpv2) - if split2 is not None: - split2 = (split2[:2], '-'.join(split2[2:])) + split2 = None + try: + split2 = cpv2.cpv + except AttributeError: + try: + split2 = _pkg_str(cpv2, eapi=eapi) + except InvalidData: + pass split_cache[cpv2] = split2 - if split1 is None or split2 is None or split1[0] != split2[0]: + if split1 is None or split2 is None or split1.cp != split2.cp: return (cpv1 > cpv2) - (cpv1 < cpv2) - return vercmp(split1[1], split2[1]) + return vercmp(split1.version, split2.version) return cmp_sort_key(cmp_cpv) def catsplit(mydep): return mydep.split("/", 1) -def best(mymatches): +def best(mymatches, eapi=None): """Accepts None arguments; assumes matches are valid.""" if not mymatches: return "" if len(mymatches) == 1: return mymatches[0] bestmatch = mymatches[0] - p2 = catpkgsplit(bestmatch)[1:] + try: + v2 = bestmatch.version + except AttributeError: + v2 = _pkg_str(bestmatch, eapi=eapi).version for x in mymatches[1:]: - p1 = catpkgsplit(x)[1:] - if pkgcmp(p1, p2) > 0: + try: + v1 = x.version + except AttributeError: + v1 = _pkg_str(x, eapi=eapi).version + if vercmp(v1, v2) > 0: bestmatch = x - p2 = catpkgsplit(bestmatch)[1:] + v2 = v1 return bestmatch diff --git a/pym/portage/xml/metadata.py b/pym/portage/xml/metadata.py index bed6a1e55d6..f820e5414b9 100644 --- a/pym/portage/xml/metadata.py +++ b/pym/portage/xml/metadata.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Provides an easy-to-use python interface to Gentoo's metadata.xml file. @@ -42,12 +42,18 @@ if sys.hexversion < 0x2070000 or \ else: try: import xml.etree.cElementTree as etree - except ImportError: + except (SystemExit, KeyboardInterrupt): + raise + except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 import xml.etree.ElementTree as etree try: from xml.parsers.expat import ExpatError -except ImportError: +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): ExpatError = SyntaxError import re diff --git a/pym/portage/xpak.py b/pym/portage/xpak.py index 86897370b0f..73f84ab75ca 100644 --- a/pym/portage/xpak.py +++ b/pym/portage/xpak.py @@ -324,7 +324,7 @@ class tbz2(object): """ self.scan() # Don't care about condition... We'll rewrite the data anyway. - if break_hardlinks and self.filestat.st_nlink > 1: + if break_hardlinks and self.filestat and self.filestat.st_nlink > 1: tmp_fname = "%s.%d" % (self.file, os.getpid()) shutil.copyfile(self.file, tmp_fname) try: @@ -360,6 +360,7 @@ class tbz2(object): def scan(self): """Scans the tbz2 to locate the xpak segment and setup internal values. This function is called by relevant functions already.""" + a = None try: mystat = os.stat(self.file) if self.filestat: @@ -378,29 +379,28 @@ class tbz2(object): self.infosize = 0 self.xpaksize = 0 if trailer[-4:] != b'STOP': - a.close() return 0 if trailer[0:8] != b'XPAKSTOP': - a.close() return 0 self.infosize = decodeint(trailer[8:12]) self.xpaksize = self.infosize + 8 a.seek(-(self.xpaksize), 2) header = a.read(16) if header[0:8] != b'XPAKPACK': - a.close() return 0 self.indexsize = decodeint(header[8:12]) self.datasize = decodeint(header[12:16]) self.indexpos = a.tell() self.index = a.read(self.indexsize) self.datapos = a.tell() - a.close() return 2 except SystemExit: raise except: return 0 + finally: + if a is not None: + a.close() def filelist(self): """Return an array of each file listed in the index.""" diff --git a/pym/repoman/checks.py b/pym/repoman/checks.py index 50c017af2ab..ca4c260b18f 100644 --- a/pym/repoman/checks.py +++ b/pym/repoman/checks.py @@ -1,16 +1,20 @@ # repoman: Checks -# Copyright 2007, 2011 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """This module contains functions used in Repoman to ascertain the quality and correctness of an ebuild.""" +import codecs +from itertools import chain import re import time import repoman.errors as errors +import portage from portage.eapi import eapi_supports_prefix, eapi_has_implicit_rdepend, \ eapi_has_src_prepare_and_src_configure, eapi_has_dosed_dohard, \ - eapi_exports_AA, eapi_exports_KV + eapi_exports_AA +from portage.const import _ENABLE_INHERIT_CHECK class LineCheck(object): """Run a check on a line of an ebuild.""" @@ -283,21 +287,35 @@ class EbuildUselessCdS(LineCheck): self.check_next_line = True class EapiDefinition(LineCheck): - """ Check that EAPI is defined before inherits""" + """ + Check that EAPI assignment conforms to PMS section 7.3.1 + (first non-comment, non-blank line). + """ repoman_check_name = 'EAPI.definition' - - eapi_re = re.compile(r'^EAPI=') - inherit_re = re.compile(r'^\s*inherit\s') + ignore_comment = True + _eapi_re = portage._pms_eapi_re def new(self, pkg): - self.inherit_line = None + self._cached_eapi = pkg.metadata['EAPI'] + self._parsed_eapi = None + self._eapi_line_num = None def check(self, num, line): - if self.eapi_re.match(line) is not None: - if self.inherit_line is not None: - return errors.EAPI_DEFINED_AFTER_INHERIT - elif self.inherit_re.match(line) is not None: - self.inherit_line = line + if self._eapi_line_num is None and line.strip(): + self._eapi_line_num = num + 1 + m = self._eapi_re.match(line) + if m is not None: + self._parsed_eapi = m.group(2) + + def end(self): + if self._parsed_eapi is None: + if self._cached_eapi != "0": + yield "valid EAPI assignment must occur on or before line: %s" % \ + self._eapi_line_num + elif self._parsed_eapi != self._cached_eapi: + yield ("bash returned EAPI '%s' which does not match " + "assignment on line: %s") % \ + (self._cached_eapi, self._eapi_line_num) class EbuildPatches(LineCheck): """Ensure ebuilds use bash arrays for PATCHES to ensure white space safety""" @@ -316,24 +334,6 @@ class EbuildQuotedA(LineCheck): if match: return "Quoted \"${A}\" on line: %d" -class EprefixifyDefined(LineCheck): - """ Check that prefix.eclass is inherited if needed""" - - repoman_check_name = 'eprefixify.defined' - - _eprefixify_re = re.compile(r'\beprefixify\b') - _inherit_prefix_re = re.compile(r'^\s*inherit\s(.*\s)?prefix\b') - - def new(self, pkg): - self._prefix_inherited = False - - def check(self, num, line): - if self._eprefixify_re.search(line) is not None: - if not self._prefix_inherited: - return errors.EPREFIXIFY_MISSING_INHERIT - elif self._inherit_prefix_re.search(line) is not None: - self._prefix_inherited = True - class NoOffsetWithHelpers(LineCheck): """ Check that the image location, the alternate root offset, and the offset prefix (D, ROOT, ED, EROOT and EPREFIX) are not used with @@ -449,43 +449,196 @@ class InheritDeprecated(LineCheck): (eclass, replacement) del self._indirect_deprecated -class InheritAutotools(LineCheck): +class InheritEclass(LineCheck): """ - Make sure appropriate functions are called in - ebuilds that inherit autotools.eclass. + Base class for checking for missing inherits, as well as excess inherits. + + Args: + eclass: Set to the name of your eclass. + funcs: A tuple of functions that this eclass provides. + comprehensive: Is the list of functions complete? + exempt_eclasses: If these eclasses are inherited, disable the missing + inherit check. """ - repoman_check_name = 'inherit.autotools' - _inherit_autotools_re = re.compile(r'^\s*inherit\s(.*\s)?autotools(\s|$)') - _autotools_funcs = ( - "eaclocal", "eautoconf", "eautoheader", - "eautomake", "eautoreconf", "_elibtoolize") - _autotools_func_re = re.compile(r'\b(' + \ - "|".join(_autotools_funcs) + r')\b') - # Exempt eclasses: - # git - An EGIT_BOOTSTRAP variable may be used to call one of - # the autotools functions. - # subversion - An ESVN_BOOTSTRAP variable may be used to call one of - # the autotools functions. - _exempt_eclasses = frozenset(["git", "subversion"]) + def __init__(self, eclass, funcs=None, comprehensive=False, + exempt_eclasses=None, ignore_missing=False, **kwargs): + self._eclass = eclass + self._comprehensive = comprehensive + self._exempt_eclasses = exempt_eclasses + self._ignore_missing = ignore_missing + inherit_re = eclass + self._inherit_re = re.compile(r'^(\s*|.*[|&]\s*)\binherit\s(.*\s)?%s(\s|$)' % inherit_re) + # Match when the function is preceded only by leading whitespace, a + # shell operator such as (, {, |, ||, or &&, or optional variable + # setting(s). This prevents false postives in things like elog + # messages, as reported in bug #413285. + self._func_re = re.compile(r'(^|[|&{(])\s*(\w+=.*)?\b(' + '|'.join(funcs) + r')\b') def new(self, pkg): - self._inherit_autotools = None - self._autotools_func_call = None - self._disabled = self._exempt_eclasses.intersection(pkg.inherited) + self.repoman_check_name = 'inherit.missing' + # We can't use pkg.inherited because that tells us all the eclass that + # have been inherited and not just the ones we inherit directly. + self._inherit = False + self._func_call = False + if self._exempt_eclasses is not None: + inherited = pkg.inherited + self._disabled = any(x in inherited for x in self._exempt_eclasses) + else: + self._disabled = False def check(self, num, line): - if self._disabled: - return - if self._inherit_autotools is None: - self._inherit_autotools = self._inherit_autotools_re.match(line) - if self._inherit_autotools is not None and \ - self._autotools_func_call is None: - self._autotools_func_call = self._autotools_func_re.search(line) + if not self._inherit: + self._inherit = self._inherit_re.match(line) + if not self._inherit: + if self._disabled or self._ignore_missing: + return + s = self._func_re.search(line) + if s: + self._func_call = True + return '%s.eclass is not inherited, but "%s" found at line: %s' % \ + (self._eclass, s.group(3), '%d') + elif not self._func_call: + self._func_call = self._func_re.search(line) def end(self): - if self._inherit_autotools and self._autotools_func_call is None: - yield 'no eauto* function called' + if not self._disabled and self._comprehensive and self._inherit and not self._func_call: + self.repoman_check_name = 'inherit.unused' + yield 'no function called from %s.eclass; please drop' % self._eclass + +# eclasses that export ${ECLASS}_src_(compile|configure|install) +_eclass_export_functions = ( + 'ant-tasks', 'apache-2', 'apache-module', 'aspell-dict', + 'autotools-utils', 'base', 'bsdmk', 'cannadic', + 'clutter', 'cmake-utils', 'db', 'distutils', 'elisp', + 'embassy', 'emboss', 'emul-linux-x86', 'enlightenment', + 'font-ebdftopcf', 'font', 'fox', 'freebsd', 'freedict', + 'games', 'games-ggz', 'games-mods', 'gdesklets', + 'gems', 'gkrellm-plugin', 'gnatbuild', 'gnat', 'gnome2', + 'gnome-python-common', 'gnustep-base', 'go-mono', 'gpe', + 'gst-plugins-bad', 'gst-plugins-base', 'gst-plugins-good', + 'gst-plugins-ugly', 'gtk-sharp-module', 'haskell-cabal', + 'horde', 'java-ant-2', 'java-pkg-2', 'java-pkg-simple', + 'java-virtuals-2', 'kde4-base', 'kde4-meta', 'kernel-2', + 'latex-package', 'linux-mod', 'mozlinguas', 'myspell', + 'myspell-r2', 'mysql', 'mysql-v2', 'mythtv-plugins', + 'oasis', 'obs-service', 'office-ext', 'perl-app', + 'perl-module', 'php-ext-base-r1', 'php-ext-pecl-r2', + 'php-ext-source-r2', 'php-lib-r1', 'php-pear-lib-r1', + 'php-pear-r1', 'python-distutils-ng', 'python', + 'qt4-build', 'qt4-r2', 'rox-0install', 'rox', 'ruby', + 'ruby-ng', 'scsh', 'selinux-policy-2', 'sgml-catalog', + 'stardict', 'sword-module', 'tetex-3', 'tetex', + 'texlive-module', 'toolchain-binutils', 'toolchain', + 'twisted', 'vdr-plugin-2', 'vdr-plugin', 'vim', + 'vim-plugin', 'vim-spell', 'virtuoso', 'vmware', + 'vmware-mod', 'waf-utils', 'webapp', 'xemacs-elisp', + 'xemacs-packages', 'xfconf', 'x-modular', 'xorg-2', + 'zproduct' +) + +_eclass_info = { + 'autotools': { + 'funcs': ( + 'eaclocal', 'eautoconf', 'eautoheader', + 'eautomake', 'eautoreconf', '_elibtoolize', + 'eautopoint' + ), + 'comprehensive': True, + + # Exempt eclasses: + # git - An EGIT_BOOTSTRAP variable may be used to call one of + # the autotools functions. + # subversion - An ESVN_BOOTSTRAP variable may be used to call one of + # the autotools functions. + 'exempt_eclasses': ('git', 'git-2', 'subversion', 'autotools-utils') + }, + + 'eutils': { + 'funcs': ( + 'estack_push', 'estack_pop', 'eshopts_push', 'eshopts_pop', + 'eumask_push', 'eumask_pop', 'epatch', 'epatch_user', + 'emktemp', 'edos2unix', 'in_iuse', 'use_if_iuse', 'usex', + 'makeopts_jobs' + ), + 'comprehensive': False, + + # These are "eclasses are the whole ebuild" type thing. + 'exempt_eclasses': _eclass_export_functions, + }, + + 'flag-o-matic': { + 'funcs': ( + 'filter-(ld)?flags', 'strip-flags', 'strip-unsupported-flags', + 'append-((ld|c(pp|xx)?))?flags', 'append-libs', + ), + 'comprehensive': False + }, + + 'libtool': { + 'funcs': ( + 'elibtoolize', + ), + 'comprehensive': True, + 'exempt_eclasses': ('autotools',) + }, + + 'multilib': { + 'funcs': ( + 'get_libdir', + ), + + # These are "eclasses are the whole ebuild" type thing. + 'exempt_eclasses': _eclass_export_functions + ('autotools', 'libtool'), + + 'comprehensive': False + }, + + 'prefix': { + 'funcs': ( + 'eprefixify', + ), + 'comprehensive': True + }, + + 'toolchain-funcs': { + 'funcs': ( + 'gen_usr_ldscript', + ), + 'comprehensive': False + }, + + 'user': { + 'funcs': ( + 'enewuser', 'enewgroup', + 'egetent', 'egethome', 'egetshell', 'esethome' + ), + 'comprehensive': True + } +} + +if not _ENABLE_INHERIT_CHECK: + # Since the InheritEclass check is experimental, in the stable branch + # we emulate the old eprefixify.defined and inherit.autotools checks. + _eclass_info = { + 'autotools': { + 'funcs': ( + 'eaclocal', 'eautoconf', 'eautoheader', + 'eautomake', 'eautoreconf', '_elibtoolize', + 'eautopoint' + ), + 'comprehensive': True, + 'ignore_missing': True, + 'exempt_eclasses': ('git', 'git-2', 'subversion', 'autotools-utils') + }, + + 'prefix': { + 'funcs': ( + 'eprefixify', + ), + 'comprehensive': False + } + } class IUseUndefined(LineCheck): """ @@ -648,33 +801,41 @@ class Eapi4GoneVars(LineCheck): class PortageInternal(LineCheck): repoman_check_name = 'portage.internal' - re = re.compile(r'[^#]*\b(ecompress|ecompressdir|env-update|prepall|prepalldocs|preplib)\b') + ignore_comment = True + # Match when the command is preceded only by leading whitespace or a shell + # operator such as (, {, |, ||, or &&. This prevents false postives in + # things like elog messages, as reported in bug #413285. + re = re.compile(r'^(\s*|.*[|&{(]+\s*)\b(ecompress|ecompressdir|env-update|prepall|prepalldocs|preplib)\b') def check(self, num, line): """Run the check on line and return error if there is one""" m = self.re.match(line) if m is not None: - return ("'%s'" % m.group(1)) + " called on line: %d" + return ("'%s'" % m.group(2)) + " called on line: %d" -_constant_checks = tuple((c() for c in ( +_constant_checks = tuple(chain((c() for c in ( EbuildHeader, EbuildWhitespace, EbuildBlankLine, EbuildQuote, EbuildAssignment, Eapi3EbuildAssignment, EbuildUselessDodoc, EbuildUselessCdS, EbuildNestedDie, - EbuildPatches, EbuildQuotedA, EapiDefinition, EprefixifyDefined, - ImplicitRuntimeDeps, InheritAutotools, InheritDeprecated, IUseUndefined, + EbuildPatches, EbuildQuotedA, EapiDefinition, + ImplicitRuntimeDeps, IUseUndefined, EMakeParallelDisabled, EMakeParallelDisabledViaMAKEOPTS, NoAsNeeded, DeprecatedBindnowFlags, SrcUnpackPatches, WantAutoDefaultValue, SrcCompileEconf, Eapi3DeprecatedFuncs, NoOffsetWithHelpers, Eapi4IncompatibleFuncs, Eapi4GoneVars, BuiltWithUse, PreserveOldLib, SandboxAddpredict, PortageInternal, - DeprecatedUseq, DeprecatedHasq))) + DeprecatedUseq, DeprecatedHasq)), + (InheritEclass(k, **kwargs) for k, kwargs in _eclass_info.items()))) _here_doc_re = re.compile(r'.*\s<<[-]?(\w+)$') _ignore_comment_re = re.compile(r'^\s*#') def run_checks(contents, pkg): + unicode_escape_codec = codecs.lookup('unicode_escape') + unicode_escape = lambda x: unicode_escape_codec.decode(x)[0] checks = _constant_checks here_doc_delim = None + multiline = None for lc in checks: lc.new(pkg) @@ -688,19 +849,56 @@ def run_checks(contents, pkg): here_doc = _here_doc_re.match(line) if here_doc is not None: here_doc_delim = re.compile(r'^\s*%s$' % here_doc.group(1)) + if here_doc_delim is not None: + continue + + # Unroll multiline escaped strings so that we can check things: + # inherit foo bar \ + # moo \ + # cow + # This will merge these lines like so: + # inherit foo bar moo cow + try: + # A normal line will end in the two bytes: <\> <\n>. So decoding + # that will result in python thinking the <\n> is being escaped + # and eat the single <\> which makes it hard for us to detect. + # Instead, strip the newline (which we know all lines have), and + # append a <0>. Then when python escapes it, if the line ended + # in a <\>, we'll end up with a <\0> marker to key off of. This + # shouldn't be a problem with any valid ebuild ... + line_escaped = unicode_escape(line.rstrip('\n') + '0') + except SystemExit: + raise + except: + # Who knows what kind of crazy crap an ebuild will have + # in it -- don't allow it to kill us. + line_escaped = line + if multiline: + # Chop off the \ and \n bytes from the previous line. + multiline = multiline[:-2] + line + if not line_escaped.endswith('\0'): + line = multiline + num = multinum + multiline = None + else: + continue + else: + if line_escaped.endswith('\0'): + multinum = num + multiline = line + continue - if here_doc_delim is None: - # We're not in a here-document. - is_comment = _ignore_comment_re.match(line) is not None - for lc in checks: - if is_comment and lc.ignore_comment: - continue - if lc.check_eapi(pkg.metadata['EAPI']): - ignore = lc.ignore_line - if not ignore or not ignore.match(line): - e = lc.check(num, line) - if e: - yield lc.repoman_check_name, e % (num + 1) + # Finally we have a full line to parse. + is_comment = _ignore_comment_re.match(line) is not None + for lc in checks: + if is_comment and lc.ignore_comment: + continue + if lc.check_eapi(pkg.metadata['EAPI']): + ignore = lc.ignore_line + if not ignore or not ignore.match(line): + e = lc.check(num, line) + if e: + yield lc.repoman_check_name, e % (num + 1) for lc in checks: i = lc.end() diff --git a/pym/repoman/errors.py b/pym/repoman/errors.py index 3209243165d..c515502c456 100644 --- a/pym/repoman/errors.py +++ b/pym/repoman/errors.py @@ -19,7 +19,6 @@ EAPI_DEFINED_AFTER_INHERIT = 'EAPI defined after inherit on line: %d' NO_AS_NEEDED = 'Upstream asneeded linking bug (no-as-needed on line: %d)' PRESERVE_OLD_LIB = 'Upstream ABI change workaround on line: %d' BUILT_WITH_USE = 'built_with_use on line: %d' -EPREFIXIFY_MISSING_INHERIT = "prefix.eclass is not inherited, but eprefixify is used on line: %d" NO_OFFSET_WITH_HELPERS = "Helper function is used with D, ROOT, ED, EROOT or EPREFIX on line :%d" SANDBOX_ADDPREDICT = 'Ebuild calls addpredict on line: %d' USEQ_ERROR = 'Ebuild calls deprecated useq function on line: %d' diff --git a/pym/repoman/herdbase.py b/pym/repoman/herdbase.py index 8ce36a704a5..fcf58b36cf0 100644 --- a/pym/repoman/herdbase.py +++ b/pym/repoman/herdbase.py @@ -1,13 +1,17 @@ # -*- coding: utf-8 -*- # repoman: Herd database analysis -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 or later import errno import xml.etree.ElementTree try: from xml.parsers.expat import ExpatError -except ImportError: +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 # This means that python is built without xml support. # We tolerate global scope import failures for optional # modules, so that ImportModulesTestCase can succeed (or diff --git a/pym/repoman/utilities.py b/pym/repoman/utilities.py index bee67aaa6b7..013858a6d87 100644 --- a/pym/repoman/utilities.py +++ b/pym/repoman/utilities.py @@ -681,7 +681,7 @@ def get_committer_name(env=None): return user def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, - new=(), removed=(), changed=(), pretend=False): + new=(), removed=(), changed=(), pretend=False, quiet=False): """ Write an entry to an existing ChangeLog, or create a new one. Updates copyright year on changed files, and updates the header of @@ -689,8 +689,8 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, """ if '<root@' in user: - err = 'Please set ECHANGELOG_USER or run as non-root' - logging.critical(err) + if not quiet: + logging.critical('Please set ECHANGELOG_USER or run as non-root') return None # ChangeLog times are in UTC @@ -711,24 +711,13 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, old_header_lines = [] header_lines = [] + clold_file = None try: clold_file = io.open(_unicode_encode(cl_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') except EnvironmentError: - clold_file = None - - clskel_file = None - if clold_file is None: - # we will only need the ChangeLog skeleton if there is no - # ChangeLog yet - try: - clskel_file = io.open(_unicode_encode(skel_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - except EnvironmentError: - pass + pass f, clnew_path = mkstemp() @@ -736,17 +725,35 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, try: if clold_file is not None: # retain header from old ChangeLog + first_line = True for line in clold_file: - line_strip = line.strip() + line_strip = line.strip() if line_strip and line[:1] != "#": clold_lines.append(line) break + # always make sure cat/pkg is up-to-date in case we are + # moving packages around, or copied from another pkg, or ... + if first_line: + if line.startswith('# ChangeLog for'): + line = '# ChangeLog for %s/%s\n' % (category, package) + first_line = False old_header_lines.append(line) header_lines.append(_update_copyright_year(year, line)) if not line_strip: break - elif clskel_file is not None: + clskel_file = None + if not header_lines: + # delay opening this until we find we need a header + try: + clskel_file = io.open(_unicode_encode(skel_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + except EnvironmentError: + pass + + if clskel_file is not None: # read skel.ChangeLog up to first empty line for line in clskel_file: line_strip = line.strip() @@ -800,13 +807,15 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, for line in textwrap.wrap(msg, 80, \ initial_indent=' ', subsequent_indent=' '): clnew_lines.append(_unicode_decode('%s\n' % line)) - clnew_lines.append(_unicode_decode('\n')) + # Don't append a trailing newline if the file is new. + if clold_file is not None: + clnew_lines.append(_unicode_decode('\n')) f = io.open(f, mode='w', encoding=_encodings['repo.content'], errors='backslashreplace') for line in clnew_lines: - f.write(line) + f.write(_unicode_decode(line)) # append stuff from old ChangeLog if clold_file is not None: @@ -832,17 +841,20 @@ def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, # in the unified_diff call below. clold_lines = old_header_lines + clold_lines - for line in clold_file: - f.write(line) + # Trim any trailing newlines. + lines = clold_file.readlines() clold_file.close() + while lines and lines[-1] == '\n': + del lines[-1] + f.writelines(lines) f.close() - # show diff (do we want to keep on doing this, or only when - # pretend?) - for line in difflib.unified_diff(clold_lines, clnew_lines, - fromfile=cl_path, tofile=cl_path, n=0): - util.writemsg_stdout(line, noiselevel=-1) - util.writemsg_stdout("\n", noiselevel=-1) + # show diff + if not quiet: + for line in difflib.unified_diff(clold_lines, clnew_lines, + fromfile=cl_path, tofile=cl_path, n=0): + util.writemsg_stdout(line, noiselevel=-1) + util.writemsg_stdout("\n", noiselevel=-1) if pretend: # remove what we've done diff --git a/runtests.sh b/runtests.sh index 56aa2cc1e31..f65bb619f79 100755 --- a/runtests.sh +++ b/runtests.sh @@ -2,7 +2,7 @@ # Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -PYTHON_VERSIONS="2.6 2.7 2.7-pypy-1.8 3.1 3.2 3.3" +PYTHON_VERSIONS="2.6 2.7 2.7-pypy-1.8 2.7-pypy-1.9 3.1 3.2 3.3" # has to be run from portage root dir cd "${0%/*}" || exit 1 |