backend changes (2nd attempt, this time it works without getbinpkg):

     * portage binpkg fetcher seem to be buggy and unreliable since v2.3.40
       also, it's weird they implemented it as an os.call to wget, instead
       of a proper python method
     * implement our own binpkg fetcher, written in python, and bypass
       buggy portage one
     * as a drawback, installation time may become slightly longer
       portage downloads in background and starts installation right away
       our new method will allways wait for the download to be complete
       before it starts to install...but hey, at least I won't get bugged
       every day about portage bugs

1 files changed, 67 insertions, 8 deletions

diff --git a/src/backend/libsisyphus.py b/src/backend/libsisyphus.py
index 1f15140..ff8dd14 100755
--- a/src/backend/libsisyphus.py
+++ b/src/backend/libsisyphus.py
@@ -9,9 +9,11 @@ import subprocess
 import sys
 import urllib3
 import io
+import wget
 from dateutil import parser
 
-redcore_portage_config_path = '/opt/redcore-build'
+portageCfg = '/opt/redcore-build/'
+portageCache = '/var/cache/packages/'
 remotePkgsDB = '/var/lib/sisyphus/csv/remotePackagesPre.csv'
 remoteDscsDB = '/var/lib/sisyphus/csv/remoteDescriptionsPre.csv'
 localPkgsDB = '/var/lib/sisyphus/csv/localPackagesPre.csv'
@@ -27,14 +29,14 @@ def checkRoot():
 # only run in binary mode (binmode) or hybrid mode (mixedmode) (CLI + GUI frontend)
 
 def checkSystemMode():
-    portage_binmode_make_conf = '/opt/redcore-build/conf/intel/portage/make.conf.amd64-binmode'
-    portage_make_conf_symlink = '/etc/portage/make.conf'
+    portageBinCfg = '/opt/redcore-build/conf/intel/portage/make.conf.amd64-binmode'
+    portageCfgSym = '/etc/portage/make.conf'
 
-    if not os.path.islink(portage_make_conf_symlink):
+    if not os.path.islink(portageCfgSym):
         print("\nmake.conf is not a symlink, refusing to run!\n")
         sys.exit(1)
     else:
-        if os.path.realpath(portage_make_conf_symlink) == portage_binmode_make_conf:
+        if os.path.realpath(portageCfgSym) == portageBinCfg:
             pass
         else:
             print("\nThe system is not set to binmode, refusing to run!\n")
@@ -42,20 +44,34 @@ def checkSystemMode():
 
 # get current mirror information, so we know where we download from (CLI + GUI frontend)
 
+def getBinhostURL():
+    binhostURL = []
+
+    portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE)
+    for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"):
+        if "PORTAGE_BINHOST" in portageOutput.rstrip():
+            binhostURL = str(portageOutput.rstrip().split("=")[1].strip('\"'))
+
+    return binhostURL
+
 def getRemotePkgsURL():
     remotePkgsURL = []
+
     portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE)
     for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"):
         if "PORTAGE_BINHOST" in portageOutput.rstrip():
             remotePkgsURL = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remotePackagesPre.csv')
+
     return remotePkgsURL
 
 def getRemoteDscsURL():
     remoteDscsURL = []
+
     portageExec = subprocess.Popen(['emerge', '--info', '--verbose'], stdout=subprocess.PIPE)
     for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"):
         if "PORTAGE_BINHOST" in portageOutput.rstrip():
             remoteDscsURL = str(portageOutput.rstrip().split("=")[1].strip('\"').replace('packages', 'csv') + 'remoteDescriptionsPre.csv')
+
     return remoteDscsURL
 
 # download remote CSV's to be imported into the database (CLI + GUI frontend)
@@ -121,7 +137,7 @@ def syncPortageTree():
 # sync portage configuration files (CLI + GUI frontend)
 
 def syncPortageCfg():
-    os.chdir(redcore_portage_config_path)
+    os.chdir(portageCfg)
     subprocess.call(['git', 'pull', '--quiet'])
 
 # check remote timestamps...if newer than local timestamps, sync everything (CLI + GUI frontend)
@@ -184,11 +200,54 @@ def rescueDB():
     syncRemoteDatabase()
     syncLocalDatabase()
 
-# call portage to install the package(s) (CLI frontend)
+# call portage to solve package(s) dependencies (CLI frontend)
+
+@animation.wait('resolving dependencies')
+def solvePkgDeps(pkgList):
+    pkgDeps = []
+    portageExec = subprocess.Popen(['emerge', '-gpq'] + pkgList, stdout=subprocess.PIPE)
+    for portageOutput in io.TextIOWrapper(portageExec.stdout, encoding="utf-8"):
+        if "/" in portageOutput.rstrip():
+            pkgDep = str(portageOutput.rstrip().split("]")[1].strip("\ "))
+            pkgDeps.append(pkgDep)
+    return pkgDeps
+
+# fetch binaries and call portage to install the package(s) from local cache (CLI frontend)
 
 def startInstall(pkgList):
     syncAll()
-    portageExec = subprocess.Popen(['emerge', '-aq'] + pkgList)
+
+    binhostURL = getBinhostURL()
+    pkgDeps = solvePkgDeps(pkgList)
+    pkgBins = []
+
+    if input("Would you like to merge these packages?" + " " + str(pkgDeps) + " " + "[y/N]" + " ").lower().strip()[:1] == "y":
+        for index, url in enumerate([binhostURL + package + '.tbz2' for package in pkgDeps]):
+            print(">>> Fetching" + " " + url)
+            wget.download(url)
+            print("\n")
+    else:
+        sys.exit("\n" + "Quitting!")
+
+    for index, binpkg in enumerate(pkgDeps):
+        pkgBin = str(binpkg.rstrip().split("/")[1])
+        pkgBins.append(pkgBin)
+
+    for index, binpkg in enumerate(pkgBins):
+        subprocess.call(['qtbz2', '-x'] + str(binpkg + '.tbz2').split())
+        CATEGORY = subprocess.check_output(['qxpak', '-x', '-O'] + str(binpkg + '.xpak').split() + ['CATEGORY'])
+        os.remove(str(binpkg + '.xpak')) # we extracted the categories, safe to delete
+
+        if os.path.isdir(portageCache + CATEGORY.decode().strip()):
+            shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2'))))
+        else:
+            os.makedirs(portageCache + CATEGORY.decode().strip())
+            shutil.move(str(binpkg + '.tbz2'), os.path.join(portageCache + CATEGORY.decode().strip(), os.path.basename(str(binpkg + '.tbz2'))))
+
+        if os.path.exists(str(binpkg + '.tbz2')):
+            os.remove(str(binpkg + '.tbz2')) # we moved the binaries in cache, safe to delete
+
+    portageExec = subprocess.Popen(['emerge', '-Kq'] + pkgList)
     portageExec.wait()
     syncLocalDatabase()