diff options
Diffstat (limited to 'eclass')
| -rw-r--r-- | eclass/Manifest.gz | bin | 37136 -> 37296 bytes | |||
| -rw-r--r-- | eclass/tests/tests-common.sh | 7 | ||||
| -rwxr-xr-x | eclass/tests/unpacker.sh | 291 | ||||
| -rw-r--r-- | eclass/unpacker.eclass | 179 |
4 files changed, 422 insertions, 55 deletions
diff --git a/eclass/Manifest.gz b/eclass/Manifest.gz Binary files differindex 9a9f1ef321f4..45c6c8ad25d7 100644 --- a/eclass/Manifest.gz +++ b/eclass/Manifest.gz diff --git a/eclass/tests/tests-common.sh b/eclass/tests/tests-common.sh index a677842b6ac5..45b1e20b933a 100644 --- a/eclass/tests/tests-common.sh +++ b/eclass/tests/tests-common.sh @@ -60,6 +60,13 @@ die() { exit 1 } +assert() { + local x pipestatus=${PIPESTATUS[*]} + for x in ${pipestatus} ; do + [[ ${x} -eq 0 ]] || die "$@" + done +} + has_version() { while [[ $1 == -* ]]; do shift diff --git a/eclass/tests/unpacker.sh b/eclass/tests/unpacker.sh new file mode 100755 index 000000000000..b15953966f65 --- /dev/null +++ b/eclass/tests/unpacker.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +# Copyright 2022 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +source tests-common.sh || exit + +inherit unpacker + +# silence the output +unpack_banner() { :; } + +TESTFILE=test.in +TESTDIR=$(mktemp -d || die) +trap 'cd - >/dev/null && rm -r "${TESTDIR}"' EXIT + +# prepare some test data +# NB: we need something "compressible", as compress(1) will return +# an error if the file "is larger than before compression" +cp ../unpacker.eclass "${TESTDIR}/${TESTFILE}" || die +cd "${TESTDIR}" || die + +test_unpack() { + local archive=${1} + local unpacked=${2} + local deps=${3} + local packcmd=${4} + + local x + for x in ${deps}; do + if ! type "${x}" &>/dev/null; then + ewarn "Skipping ${archive}, tool ${x} not found" + return + fi + done + + rm -rf testdir || die + mkdir -p testdir || die + + tbegin "unpacking ${archive}" + eval "${packcmd}" + assert "packing ${archive} failed" + cd testdir || die + local out + out=$( + _unpacker "../${archive}" 2>&1 + ) + ret=$? + if [[ ${ret} -eq 0 ]]; then + if [[ ! -f ${unpacked} ]]; then + eerror "${unpacked} not found after unpacking" + ret=1 + elif ! diff -u "${unpacked}" "../${TESTFILE}"; then + eerror "${unpacked} different than input" + ret=1 + fi + fi + [[ ${ret} -ne 0 ]] && echo "${out}" >&2 + tend ${ret} + + cd .. || die + rm -f "${archive}" || die +} + +test_compressed_file() { + local suffix=${1} + local tool=${2} + + test_unpack "test${suffix}" test "${tool}" \ + "${tool} -c \${TESTFILE} > \${archive}" +} + +test_compressed_file_multistream() { + local suffix=${1} + local tool=${2} + + test_unpack "test+multistream${suffix}" "test+multistream" "${tool}" \ + "head -n 300 \${TESTFILE} | ${tool} -c > \${archive} && + tail -n +301 \${TESTFILE} | ${tool} -c >> \${archive}" +} + +test_compressed_file_with_junk() { + local suffix=${1} + local tool=${2} + local flag=${3} + + test_unpack "test+junk${suffix}" "test+junk" "${tool}" \ + "${tool} -c \${TESTFILE} > \${archive} && cat test.in >> \${archive}" +} + +test_compressed_tar() { + local suffix=${1} + local tool=${2} + + test_unpack "test${suffix}" test.in "tar ${tool}" \ + "tar -c \${TESTFILE} | ${tool} -c > \${archive}" +} + +test_compressed_cpio() { + local suffix=${1} + local tool=${2} + + test_unpack "test${suffix}" test.in "cpio ${tool}" \ + "cpio -o --quiet <<<\${TESTFILE} | ${tool} -c > \${archive}" +} + +create_deb() { + local suffix=${1} + local tool=${2} + local archive=${3} + local infile=${4} + + echo 2.0 > debian-binary || die + : > control || die + tar -cf control.tar control || die + tar -c "${infile}" | ${tool} > "data.tar${suffix}" + assert "packing data.tar${suffix} failed" + ar r "${archive}" debian-binary control.tar "data.tar${suffix}" \ + 2>/dev/null || die + rm -f control control.tar "data.tar${suffix}" debian-binary || die +} + +test_deb() { + local suffix=${1} + local tool=${2} + local tool_cmd + + if [[ -n ${tool} ]]; then + tool_cmd="${tool} -c" + else + tool_cmd=cat + fi + + test_unpack "test-${tool}_1.2.3_noarch.deb" test.in "ar tar ${tool}" \ + "create_deb '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}" + # also test with the handwoven implementation used on Prefix + EPREFIX=/foo \ + test_unpack "test_pfx-${tool}_1.2.3_noarch.deb" test.in "ar tar ${tool}" \ + "create_deb '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}" +} + +create_gpkg() { + local suffix=${1} + local tool=${2} + local archive=${3} + local infile=${4} + local gpkg_dir=${archive%.gpkg.tar} + + mkdir image metadata "${gpkg_dir}" || die + cp "${infile}" image/ || die + tar -c metadata | ${tool} > "${gpkg_dir}/metadata.tar${suffix}" + assert "packing metadata.tar${suffix} failed" + : > "${gpkg_dir}/metadata.tar${suffix}.sig" || die + tar -c image | ${tool} > "${gpkg_dir}/image.tar${suffix}" + assert "packing image.tar${suffix} failed" + : > "${gpkg_dir}/image.tar${suffix}.sig" || die + : > "${gpkg_dir}"/gpkg-1 || die + tar -cf "${archive}" --format=ustar \ + "${gpkg_dir}"/{gpkg-1,{metadata,image}.tar"${suffix}"} || die + rm -r image metadata "${gpkg_dir}" || die +} + +test_gpkg() { + local suffix=${1} + local tool=${2} + local tool_cmd + + if [[ -n ${tool} ]]; then + tool_cmd="${tool} -c" + else + tool_cmd=cat + fi + + test_unpack "test-${tool}-1.2.3-1.gpkg.tar" \ + "test-${tool}-1.2.3-1/image/test.in" "tar ${tool}" \ + "create_gpkg '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}" +} + +test_reject_junk() { + local suffix=${1} + local archive=test${1} + + rm -rf testdir || die + mkdir -p testdir || die + + tbegin "rejecting junk named ${archive}" + cat test.in >> "${archive}" || die + cd testdir || die + ( + # some decompressors (e.g. cpio) are very verbose about junk + _unpacker "../${archive}" &>/dev/null + ) + [[ $? -ne 0 ]] + ret=$? + tend ${ret} + + cd .. || die + rm -f "${archive}" || die +} + +test_compressed_file .bz2 bzip2 +test_compressed_file .Z compress +test_compressed_file .gz gzip +test_compressed_file .lzma lzma +test_compressed_file .xz xz +test_compressed_file .lz lzip +test_compressed_file .zst zstd +test_compressed_file .lz4 lz4 +test_compressed_file .lzo lzop + +test_compressed_file_multistream .bz2 bzip2 +test_compressed_file_multistream .gz gzip +test_compressed_file_multistream .xz xz +test_compressed_file_multistream .lz lzip +test_compressed_file_multistream .zst zstd + +test_compressed_file_with_junk .bz2 bzip2 +test_compressed_file_with_junk .lz lzip + +test_unpack test.tar test.in tar 'tar -cf ${archive} ${TESTFILE}' +test_compressed_tar .tar.bz2 bzip2 +test_compressed_tar .tbz bzip2 +test_compressed_tar .tbz2 bzip2 +test_compressed_tar .tar.Z compress +test_compressed_tar .tar.gz gzip +test_compressed_tar .tgz gzip +test_compressed_tar .tar.lzma lzma +test_compressed_tar .tar.xz xz +test_compressed_tar .txz xz +test_compressed_tar .tar.lz lzip +test_compressed_tar .tar.zst zstd +test_compressed_tar .tar.lz4 lz4 +test_compressed_tar .tar.lzo lzop + +test_unpack test.cpio test.in cpio 'cpio -o --quiet <<<${TESTFILE} > ${archive}' +test_compressed_cpio .cpio.bz2 bzip2 +test_compressed_cpio .cpio.Z compress +test_compressed_cpio .cpio.gz gzip +test_compressed_cpio .cpio.lzma lzma +test_compressed_cpio .cpio.xz xz +test_compressed_cpio .cpio.lz lzip +test_compressed_cpio .cpio.zst zstd +test_compressed_cpio .cpio.lz4 lz4 +test_compressed_cpio .cpio.lzo lzop + +test_deb +test_deb .gz gzip +test_deb .xz xz +test_deb .bz2 bzip2 +test_deb .lzma lzma + +test_gpkg +test_gpkg .gz gzip +test_gpkg .bz2 bzip2 +test_gpkg .lz4 lz4 +test_gpkg .lz lzip +test_gpkg .lzma lzma +test_gpkg .lzo lzop +test_gpkg .xz xz +test_gpkg .zst zstd + +test_unpack test.zip test.in zip 'zip -q ${archive} ${TESTFILE}' +# test handling non-adjusted zip with junk prepended +test_unpack test.zip test.in zip \ + 'zip -q testdir/tmp.zip ${TESTFILE} && cat test.in testdir/tmp.zip > ${archive}' +test_unpack test.7z test.in 7z '7z -bso0 a ${archive} ${TESTFILE}' +test_unpack test.lha test.in lha 'lha a -q ${archive} ${TESTFILE}' +test_unpack test.lzh test.in lha 'lha a -q ${archive} ${TESTFILE}' +test_unpack test.rar test.in rar 'rar -idq a ${archive} ${TESTFILE}' + +# TODO: .run/.sh/.bin + +test_reject_junk .bz2 +test_reject_junk .Z +test_reject_junk .gz +test_reject_junk .lzma +test_reject_junk .xz +test_reject_junk .lz +test_reject_junk .zst +test_reject_junk .tar +test_reject_junk .cpio +test_reject_junk .gpkg.tar +test_reject_junk .deb +test_reject_junk .zip +test_reject_junk .7z +test_reject_junk .rar +test_reject_junk .lha +test_reject_junk .lzh + +texit diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass index f6e83c53bf23..3d23151b636e 100644 --- a/eclass/unpacker.eclass +++ b/eclass/unpacker.eclass @@ -4,7 +4,7 @@ # @ECLASS: unpacker.eclass # @MAINTAINER: # base-system@gentoo.org -# @SUPPORTED_EAPIS: 5 6 7 8 +# @SUPPORTED_EAPIS: 6 7 8 # @BLURB: helpers for extraneous file formats and consistent behavior across EAPIs # @DESCRIPTION: # Some extraneous file formats are not part of PMS, or are only in certain @@ -16,21 +16,22 @@ # - support partial unpacks? case ${EAPI:-0} in - [5678]) ;; + [678]) ;; *) die "${ECLASS}: EAPI ${EAPI:-0} not supported" ;; esac if [[ -z ${_UNPACKER_ECLASS} ]]; then _UNPACKER_ECLASS=1 -inherit toolchain-funcs +inherit multiprocessing toolchain-funcs # @ECLASS_VARIABLE: UNPACKER_BZ2 # @USER_VARIABLE # @DEFAULT_UNSET # @DESCRIPTION: # Utility to use to decompress bzip2 files. Will dynamically pick between -# `pbzip2` and `bzip2`. Make sure your choice accepts the "-dc" options. +# `lbzip2`, `pbzip2` and `bzip2`. Make sure your choice accepts the "-dc" +# options. # Note: this is meant for users to set, not ebuilds. # @ECLASS_VARIABLE: UNPACKER_LZIP @@ -272,31 +273,39 @@ unpack_deb() { unpack_banner "${deb}" - # on AIX ar doesn't work out as their ar used a different format - # from what GNU ar (and thus what .deb files) produce - if [[ -n ${EPREFIX} ]] ; then - { - read # global header - [[ ${REPLY} = "!<arch>" ]] || die "${deb} does not seem to be a deb archive" - local f timestamp uid gid mode size magic - while read f timestamp uid gid mode size magic ; do - [[ -n ${f} && -n ${size} ]] || continue # ignore empty lines - if [[ ${f} = "data.tar"* ]] ; then - head -c "${size}" > "${f}" - else - head -c "${size}" > /dev/null # trash it - fi - done - } < "${deb}" - else - $(tc-getBUILD_AR) x "${deb}" || die - fi - - unpacker ./data.tar* - - # Clean things up #458658. No one seems to actually care about - # these, so wait until someone requests to do something else ... - rm -f debian-binary {control,data}.tar* + { + # on AIX ar doesn't work out as their ar used a different format + # from what GNU ar (and thus what .deb files) produce + if [[ -n ${EPREFIX} ]] ; then + { + read # global header + [[ ${REPLY} = "!<arch>" ]] || die "${deb} does not seem to be a deb archive" + local f timestamp uid gid mode size magic + while read f timestamp uid gid mode size magic ; do + [[ -n ${f} && -n ${size} ]] || continue # ignore empty lines + # GNU ar uses / as filename terminator (and .deb permits that) + f=${f%/} + if [[ ${f} = "data.tar"* ]] ; then + local decomp=$(_unpacker_get_decompressor "${f}") + head -c "${size}" | ${decomp:-cat} + assert "unpacking ${f} from ${deb} failed" + break + else + head -c "${size}" > /dev/null # trash it + fi + done + } < "${deb}" + else + local f=$( + $(tc-getBUILD_AR) t "${deb}" | grep ^data.tar + assert "data not found in ${deb}" + ) + local decomp=$(_unpacker_get_decompressor "${f}") + $(tc-getBUILD_AR) p "${deb}" "${f}" | ${decomp:-cat} + assert "unpacking ${f} from ${deb} failed" + fi + } | tar --no-same-owner -x + assert "unpacking ${deb} failed" } # @FUNCTION: unpack_cpio @@ -344,8 +353,11 @@ unpack_7z() { local p7z=$(find_unpackable_file "$1") unpack_banner "${p7z}" - local output="$(7z x -y "${p7z}")" + # warning: putting local and command substitution in a single call + # discards the exit status! + local output + output="$(7z x -y "${p7z}")" if [ $? -ne 0 ]; then echo "${output}" >&2 die "unpacking ${p7z} failed (arch=unpack_7z)" @@ -376,6 +388,73 @@ unpack_lha() { lha xfq "${lha}" || die "unpacking ${lha} failed (arch=unpack_lha)" } +# @FUNCTION: _unpacker_get_decompressor +# @INTERNAL +# @USAGE: <filename> +# @DESCRIPTION: +# Get decompressor command for specified filename. +_unpacker_get_decompressor() { + case ${1} in + *.bz2|*.tbz|*.tbz2) + local bzcmd=${PORTAGE_BZIP2_COMMAND:-$( + type -P lbzip2 || type -P pbzip2 || type -P bzip2 + )} + local bzuncmd=${PORTAGE_BUNZIP2_COMMAND:-${bzcmd} -d} + : ${UNPACKER_BZ2:=${bzuncmd}} + echo "${UNPACKER_BZ2} -c" + ;; + *.z|*.gz|*.tgz) + echo "gzip -dc" ;; + *.lzma|*.xz|*.txz) + echo "xz -T$(makeopts_jobs) -dc" ;; + *.lz) + : ${UNPACKER_LZIP:=$(type -P plzip || type -P pdlzip || type -P lzip)} + echo "${UNPACKER_LZIP} -dc" ;; + *.zst) + echo "zstd -dc" ;; + *.lz4) + echo "lz4 -dc" ;; + *.lzo) + echo "lzop -dc" ;; + esac +} + +# @FUNCTION: unpack_gpkg +# @USAGE: <gpkg file> +# @DESCRIPTION: +# Unpack the image subarchive of a GPKG package on-the-fly, preserving +# the original directory structure (i.e. into <gpkg-dir>/image). +unpack_gpkg() { + [[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>" + + local gpkg=$(find_unpackable_file "$1") + unpack_banner "${gpkg}" + + local l images=() + while read -r l; do + case ${l} in + */image.tar*.sig) + ;; + */image.tar*) + images+=( "${l}" ) + ;; + esac + done < <(tar -tf "${gpkg}" || die "unable to list ${gpkg}") + + if [[ ${#images[@]} -eq 0 ]]; then + die "No image.tar found in ${gpkg}" + elif [[ ${#images[@]} -gt 1 ]]; then + die "More than one image.tar found in ${gpkg}" + fi + + local decomp=$(_unpacker_get_decompressor "${images[0]}") + local dirname=${images[0]%/*} + mkdir -p "${dirname}" || die + tar -xOf "${gpkg}" "${images[0]}" | ${decomp:-cat} | + tar --no-same-owner -xC "${dirname}" + assert "Unpacking ${gpkg} failed" +} + # @FUNCTION: _unpacker # @USAGE: <one archive to unpack> # @INTERNAL @@ -386,32 +465,17 @@ _unpacker() { [[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>" local a=$1 - local m=$(echo "${a}" | tr '[:upper:]' '[:lower:]') + local m=${a,,} a=$(find_unpackable_file "${a}") # first figure out the decompression method - local comp="" - case ${m} in - *.bz2|*.tbz|*.tbz2) - local bzcmd=${PORTAGE_BZIP2_COMMAND:-$(type -P pbzip2 || type -P bzip2)} - local bzuncmd=${PORTAGE_BUNZIP2_COMMAND:-${bzcmd} -d} - : ${UNPACKER_BZ2:=${bzuncmd}} - comp="${UNPACKER_BZ2} -c" - ;; - *.z|*.gz|*.tgz) - comp="gzip -dc" ;; - *.lzma|*.xz|*.txz) - comp="xz -dc" ;; - *.lz) - : ${UNPACKER_LZIP:=$(type -P plzip || type -P pdlzip || type -P lzip)} - comp="${UNPACKER_LZIP} -dc" ;; - *.zst) - comp="zstd -dfc" ;; - esac + local comp=$(_unpacker_get_decompressor "${m}") # then figure out if there are any archiving aspects local arch="" case ${m} in + *.gpkg.tar) + arch="unpack_gpkg" ;; *.tgz|*.tbz|*.tbz2|*.txz|*.tar.*|*.tar) arch="tar --no-same-owner -xof" ;; *.cpio.*|*.cpio) @@ -437,13 +501,13 @@ _unpacker() { esac # 7z, rar and lha/lzh are handled by package manager in EAPI < 8 - if [[ ${EAPI} != [567] ]]; then + if [[ ${EAPI} != [67] ]]; then case ${m} in *.7z) arch="unpack_7z" ;; - *.rar|*.RAR) + *.rar) arch="unpack_rar" ;; - *.LHA|*.LHa|*.lha|*.lzh) + *.lha|*.lzh) arch="unpack_lha" ;; esac fi @@ -506,10 +570,11 @@ unpacker_src_uri_depends() { fi for uri in "$@" ; do - case ${uri} in + local m=${uri,,} + case ${m} in *.cpio.*|*.cpio) d="app-arch/cpio" ;; - *.rar|*.RAR) + *.rar) d="app-arch/unrar" ;; *.7z) d="app-arch/p7zip" ;; @@ -521,8 +586,12 @@ unpacker_src_uri_depends() { d="|| ( app-arch/plzip app-arch/pdlzip app-arch/lzip )" ;; *.zst) d="app-arch/zstd" ;; - *.LHA|*.LHa|*.lha|*.lzh) + *.lha|*.lzh) d="app-arch/lha" ;; + *.lz4) + d="app-arch/lz4" ;; + *.lzo) + d="app-arch/lzop" ;; esac deps+=" ${d}" done |