aboutsummaryrefslogtreecommitdiffstats
path: root/pym/portage/getbinpkg.py
diff options
context:
space:
mode:
Diffstat (limited to 'pym/portage/getbinpkg.py')
-rw-r--r--pym/portage/getbinpkg.py255
1 files changed, 156 insertions, 99 deletions
diff --git a/pym/portage/getbinpkg.py b/pym/portage/getbinpkg.py
index 212f78889..14dc149b1 100644
--- a/pym/portage/getbinpkg.py
+++ b/pym/portage/getbinpkg.py
@@ -1,7 +1,9 @@
# getbinpkg.py -- Portage binary-package helper functions
-# Copyright 2003-2012 Gentoo Foundation
+# Copyright 2003-2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import unicode_literals
+
from portage.output import colorize
from portage.cache.mappings import slot_dict_class
from portage.localization import _
@@ -18,6 +20,7 @@ import socket
import time
import tempfile
import base64
+import warnings
_all_errors = [NotImplementedError, ValueError, socket.error]
@@ -39,7 +42,7 @@ except ImportError:
try:
import ftplib
except ImportError as e:
- sys.stderr.write(colorize("BAD","!!! CANNOT IMPORT FTPLIB: ")+str(e)+"\n")
+ sys.stderr.write(colorize("BAD", "!!! CANNOT IMPORT FTPLIB: ") + str(e) + "\n")
else:
_all_errors.extend(ftplib.all_errors)
@@ -55,24 +58,28 @@ try:
from httplib import ResponseNotReady as http_client_ResponseNotReady
from httplib import error as http_client_error
except ImportError as e:
- sys.stderr.write(colorize("BAD","!!! CANNOT IMPORT HTTP.CLIENT: ")+str(e)+"\n")
+ sys.stderr.write(colorize("BAD", "!!! CANNOT IMPORT HTTP.CLIENT: ") + str(e) + "\n")
else:
_all_errors.append(http_client_error)
_all_errors = tuple(_all_errors)
if sys.hexversion >= 0x3000000:
+ # pylint: disable=W0622
long = int
def make_metadata_dict(data):
- myid,myglob = data
+
+ warnings.warn("portage.getbinpkg.make_metadata_dict() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
+ myid, _myglob = data
mydict = {}
for k_bytes in portage.xpak.getindex_mem(myid):
k = _unicode_decode(k_bytes,
encoding=_encodings['repo.content'], errors='replace')
- if k not in _all_metadata_keys and \
- k != "CATEGORY":
+ if k not in _all_metadata_keys and k != "CATEGORY":
continue
v = _unicode_decode(portage.xpak.getitem(data, k_bytes),
encoding=_encodings['repo.content'], errors='replace')
@@ -84,13 +91,17 @@ class ParseLinks(html_parser_HTMLParser):
"""Parser class that overrides HTMLParser to grab all anchors from an html
page and provide suffix and prefix limitors"""
def __init__(self):
+
+ warnings.warn("portage.getbinpkg.ParseLinks is deprecated",
+ DeprecationWarning, stacklevel=2)
+
self.PL_anchors = []
html_parser_HTMLParser.__init__(self)
def get_anchors(self):
return self.PL_anchors
- def get_anchors_by_prefix(self,prefix):
+ def get_anchors_by_prefix(self, prefix):
newlist = []
for x in self.PL_anchors:
if x.startswith(prefix):
@@ -98,7 +109,7 @@ class ParseLinks(html_parser_HTMLParser):
newlist.append(x[:])
return newlist
- def get_anchors_by_suffix(self,suffix):
+ def get_anchors_by_suffix(self, suffix):
newlist = []
for x in self.PL_anchors:
if x.endswith(suffix):
@@ -106,10 +117,10 @@ class ParseLinks(html_parser_HTMLParser):
newlist.append(x[:])
return newlist
- def handle_endtag(self,tag):
+ def handle_endtag(self, tag):
pass
- def handle_starttag(self,tag,attrs):
+ def handle_starttag(self, tag, attrs):
if tag == "a":
for x in attrs:
if x[0] == 'href':
@@ -117,16 +128,19 @@ class ParseLinks(html_parser_HTMLParser):
self.PL_anchors.append(urllib_parse_unquote(x[1]))
-def create_conn(baseurl,conn=None):
- """(baseurl,conn) --- Takes a protocol://site:port/address url, and an
+def create_conn(baseurl, conn=None):
+ """Takes a protocol://site:port/address url, and an
optional connection. If connection is already active, it is passed on.
baseurl is reduced to address and is returned in tuple (conn,address)"""
- parts = baseurl.split("://",1)
+ warnings.warn("portage.getbinpkg.create_conn() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
+ parts = baseurl.split("://", 1)
if len(parts) != 2:
raise ValueError(_("Provided URI does not "
"contain protocol identifier. '%s'") % baseurl)
- protocol,url_parts = parts
+ protocol, url_parts = parts
del parts
url_parts = url_parts.split("/")
@@ -137,7 +151,7 @@ def create_conn(baseurl,conn=None):
address = "/"+"/".join(url_parts[1:])
del url_parts
- userpass_host = host.split("@",1)
+ userpass_host = host.split("@", 1)
if len(userpass_host) == 1:
host = userpass_host[0]
userpass = ["anonymous"]
@@ -196,10 +210,10 @@ def create_conn(baseurl,conn=None):
host = host[:-1]
conn = ftplib.FTP(host)
if password:
- conn.login(username,password)
+ conn.login(username, password)
else:
sys.stderr.write(colorize("WARN",
- _(" * No password provided for username"))+" '%s'" % \
+ _(" * No password provided for username")) + " '%s'" % \
(username,) + "\n\n")
conn.login(username)
conn.set_pasv(passive)
@@ -216,11 +230,15 @@ def create_conn(baseurl,conn=None):
else:
raise NotImplementedError(_("%s is not a supported protocol.") % protocol)
- return (conn,protocol,address, http_params, http_headers)
+ return (conn, protocol, address, http_params, http_headers)
def make_ftp_request(conn, address, rest=None, dest=None):
- """(conn,address,rest) --- uses the conn object to request the data
+ """Uses the |conn| object to request the data
from address and issuing a rest if it is passed."""
+
+ warnings.warn("portage.getbinpkg.make_ftp_request() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
try:
if dest:
@@ -235,9 +253,9 @@ def make_ftp_request(conn, address, rest=None, dest=None):
rest = 0
if rest != None:
- mysocket = conn.transfercmd("RETR "+str(address), rest)
+ mysocket = conn.transfercmd("RETR %s" % str(address), rest)
else:
- mysocket = conn.transfercmd("RETR "+str(address))
+ mysocket = conn.transfercmd("RETR %s" % str(address))
mydata = ""
while 1:
@@ -259,28 +277,31 @@ def make_ftp_request(conn, address, rest=None, dest=None):
conn.voidresp()
conn.voidcmd("TYPE A")
- return mydata,not (fsize==data_size),""
+ return mydata, (fsize != data_size), ""
except ValueError as e:
- return None,int(str(e)[:4]),str(e)
+ return None, int(str(e)[:4]), str(e)
-def make_http_request(conn, address, params={}, headers={}, dest=None):
- """(conn,address,params,headers) --- uses the conn object to request
+def make_http_request(conn, address, _params={}, headers={}, dest=None):
+ """Uses the |conn| object to request
the data from address, performing Location forwarding and using the
optional params and headers."""
+ warnings.warn("portage.getbinpkg.make_http_request() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
rc = 0
response = None
while (rc == 0) or (rc == 301) or (rc == 302):
try:
- if (rc != 0):
- conn,ignore,ignore,ignore,ignore = create_conn(address)
+ if rc != 0:
+ conn = create_conn(address)[0]
conn.request("GET", address, body=None, headers=headers)
except SystemExit as e:
raise
except Exception as e:
- return None,None,"Server request failed: "+str(e)
+ return None, None, "Server request failed: %s" % str(e)
response = conn.getresponse()
rc = response.status
@@ -289,7 +310,7 @@ def make_http_request(conn, address, params={}, headers={}, dest=None):
ignored_data = response.read()
del ignored_data
for x in str(response.msg).split("\n"):
- parts = x.split(": ",1)
+ parts = x.split(": ", 1)
if parts[0] == "Location":
if (rc == 301):
sys.stderr.write(colorize("BAD",
@@ -302,16 +323,20 @@ def make_http_request(conn, address, params={}, headers={}, dest=None):
break
if (rc != 200) and (rc != 206):
- return None,rc,"Server did not respond successfully ("+str(response.status)+": "+str(response.reason)+")"
+ return None, rc, "Server did not respond successfully (%s: %s)" % (str(response.status), str(response.reason))
if dest:
dest.write(response.read())
- return "",0,""
+ return "", 0, ""
- return response.read(),0,""
+ return response.read(), 0, ""
def match_in_array(array, prefix="", suffix="", match_both=1, allow_overlap=0):
+
+ warnings.warn("portage.getbinpkg.match_in_array() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
myarray = []
if not (prefix and suffix):
@@ -344,20 +369,22 @@ def match_in_array(array, prefix="", suffix="", match_both=1, allow_overlap=0):
continue # Doesn't match.
return myarray
-
-def dir_get_list(baseurl,conn=None):
- """(baseurl[,connection]) -- Takes a base url to connect to and read from.
+def dir_get_list(baseurl, conn=None):
+ """Takes a base url to connect to and read from.
URI should be in the form <proto>://<site>[:port]<path>
Connection is used for persistent connection instances."""
+ warnings.warn("portage.getbinpkg.dir_get_list() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
if not conn:
keepconnection = 0
else:
keepconnection = 1
- conn,protocol,address,params,headers = create_conn(baseurl, conn)
+ conn, protocol, address, params, headers = create_conn(baseurl, conn)
listing = None
if protocol in ["http","https"]:
@@ -365,7 +392,7 @@ def dir_get_list(baseurl,conn=None):
# http servers can return a 400 error here
# if the address doesn't end with a slash.
address += "/"
- page,rc,msg = make_http_request(conn,address,params,headers)
+ page, rc, msg = make_http_request(conn, address, params, headers)
if page:
parser = ParseLinks()
@@ -395,23 +422,26 @@ def dir_get_list(baseurl,conn=None):
return listing
-def file_get_metadata(baseurl,conn=None, chunk_size=3000):
- """(baseurl[,connection]) -- Takes a base url to connect to and read from.
+def file_get_metadata(baseurl, conn=None, chunk_size=3000):
+ """Takes a base url to connect to and read from.
URI should be in the form <proto>://<site>[:port]<path>
Connection is used for persistent connection instances."""
+ warnings.warn("portage.getbinpkg.file_get_metadata() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
if not conn:
keepconnection = 0
else:
keepconnection = 1
- conn,protocol,address,params,headers = create_conn(baseurl, conn)
+ conn, protocol, address, params, headers = create_conn(baseurl, conn)
if protocol in ["http","https"]:
- headers["Range"] = "bytes=-"+str(chunk_size)
- data,rc,msg = make_http_request(conn, address, params, headers)
+ headers["Range"] = "bytes=-%s" % str(chunk_size)
+ data, _x, _x = make_http_request(conn, address, params, headers)
elif protocol in ["ftp"]:
- data,rc,msg = make_ftp_request(conn, address, -chunk_size)
+ data, _x, _x = make_ftp_request(conn, address, -chunk_size)
elif protocol == "sftp":
f = conn.open(address)
try:
@@ -424,21 +454,21 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000):
if data:
xpaksize = portage.xpak.decodeint(data[-8:-4])
- if (xpaksize+8) > chunk_size:
- myid = file_get_metadata(baseurl, conn, (xpaksize+8))
+ if (xpaksize + 8) > chunk_size:
+ myid = file_get_metadata(baseurl, conn, xpaksize + 8)
if not keepconnection:
conn.close()
return myid
else:
- xpak_data = data[len(data)-(xpaksize+8):-8]
+ xpak_data = data[len(data) - (xpaksize + 8):-8]
del data
myid = portage.xpak.xsplit_mem(xpak_data)
if not myid:
- myid = None,None
+ myid = None, None
del xpak_data
else:
- myid = None,None
+ myid = None, None
if not keepconnection:
conn.close()
@@ -446,53 +476,79 @@ def file_get_metadata(baseurl,conn=None, chunk_size=3000):
return myid
-def file_get(baseurl,dest,conn=None,fcmd=None,filename=None):
- """(baseurl,dest,fcmd=) -- Takes a base url to connect to and read from.
+def file_get(baseurl=None, dest=None, conn=None, fcmd=None, filename=None,
+ fcmd_vars=None):
+ """Takes a base url to connect to and read from.
URI should be in the form <proto>://[user[:pass]@]<site>[:port]<path>"""
if not fcmd:
- return file_get_lib(baseurl,dest,conn)
- if not filename:
- filename = os.path.basename(baseurl)
-
- variables = {
- "DISTDIR": dest,
- "URI": baseurl,
- "FILE": filename
- }
+
+ warnings.warn("Use of portage.getbinpkg.file_get() without the fcmd "
+ "parameter is deprecated", DeprecationWarning, stacklevel=2)
+
+ return file_get_lib(baseurl, dest, conn)
+
+ variables = {}
+
+ if fcmd_vars is not None:
+ variables.update(fcmd_vars)
+
+ if "DISTDIR" not in variables:
+ if dest is None:
+ raise portage.exception.MissingParameter(
+ _("%s is missing required '%s' key") %
+ ("fcmd_vars", "DISTDIR"))
+ variables["DISTDIR"] = dest
+
+ if "URI" not in variables:
+ if baseurl is None:
+ raise portage.exception.MissingParameter(
+ _("%s is missing required '%s' key") %
+ ("fcmd_vars", "URI"))
+ variables["URI"] = baseurl
+
+ if "FILE" not in variables:
+ if filename is None:
+ filename = os.path.basename(variables["URI"])
+ variables["FILE"] = filename
from portage.util import varexpand
from portage.process import spawn
myfetch = portage.util.shlex_split(fcmd)
myfetch = [varexpand(x, mydict=variables) for x in myfetch]
- fd_pipes= {
- 0:sys.stdin.fileno(),
- 1:sys.stdout.fileno(),
- 2:sys.stdout.fileno()
+ fd_pipes = {
+ 0: portage._get_stdin().fileno(),
+ 1: sys.__stdout__.fileno(),
+ 2: sys.__stdout__.fileno()
}
+ sys.__stdout__.flush()
+ sys.__stderr__.flush()
retval = spawn(myfetch, env=os.environ.copy(), fd_pipes=fd_pipes)
if retval != os.EX_OK:
sys.stderr.write(_("Fetcher exited with a failure condition.\n"))
return 0
return 1
-def file_get_lib(baseurl,dest,conn=None):
- """(baseurl[,connection]) -- Takes a base url to connect to and read from.
+def file_get_lib(baseurl, dest, conn=None):
+ """Takes a base url to connect to and read from.
URI should be in the form <proto>://<site>[:port]<path>
Connection is used for persistent connection instances."""
+ warnings.warn("portage.getbinpkg.file_get_lib() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
if not conn:
keepconnection = 0
else:
keepconnection = 1
- conn,protocol,address,params,headers = create_conn(baseurl, conn)
+ conn, protocol, address, params, headers = create_conn(baseurl, conn)
- sys.stderr.write("Fetching '"+str(os.path.basename(address)+"'\n"))
- if protocol in ["http","https"]:
- data,rc,msg = make_http_request(conn, address, params, headers, dest=dest)
+ sys.stderr.write("Fetching '" + str(os.path.basename(address)) + "'\n")
+ if protocol in ["http", "https"]:
+ data, rc, _msg = make_http_request(conn, address, params, headers, dest=dest)
elif protocol in ["ftp"]:
- data,rc,msg = make_ftp_request(conn, address, dest=dest)
+ data, rc, _msg = make_ftp_request(conn, address, dest=dest)
elif protocol == "sftp":
rc = 0
try:
@@ -522,8 +578,10 @@ def file_get_lib(baseurl,dest,conn=None):
def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=1, makepickle=None):
- """(baseurl,conn,chunk_size,verbose) --
- """
+
+ warnings.warn("portage.getbinpkg.dir_get_metadata() is deprecated",
+ DeprecationWarning, stacklevel=2)
+
if not conn:
keepconnection = 0
else:
@@ -536,7 +594,7 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
makepickle = "/var/cache/edb/metadata.idx.most_recent"
try:
- conn, protocol, address, params, headers = create_conn(baseurl, conn)
+ conn = create_conn(baseurl, conn)[0]
except _all_errors as e:
# ftplib.FTP(host) can raise errors like this:
# socket.error: (111, 'Connection refused')
@@ -557,18 +615,20 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
out.write(_("Loaded metadata pickle.\n"))
out.flush()
metadatafile.close()
- except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError):
+ except (SystemExit, KeyboardInterrupt):
+ raise
+ except Exception:
metadata = {}
if baseurl not in metadata:
- metadata[baseurl]={}
+ metadata[baseurl] = {}
if "indexname" not in metadata[baseurl]:
- metadata[baseurl]["indexname"]=""
+ metadata[baseurl]["indexname"] = ""
if "timestamp" not in metadata[baseurl]:
- metadata[baseurl]["timestamp"]=0
+ metadata[baseurl]["timestamp"] = 0
if "unmodified" not in metadata[baseurl]:
- metadata[baseurl]["unmodified"]=0
+ metadata[baseurl]["unmodified"] = 0
if "data" not in metadata[baseurl]:
- metadata[baseurl]["data"]={}
+ metadata[baseurl]["data"] = {}
if not os.access(cache_path, os.W_OK):
sys.stderr.write(_("!!! Unable to write binary metadata to disk!\n"))
@@ -594,36 +654,36 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
for mfile in metalist:
if usingcache and \
((metadata[baseurl]["indexname"] != mfile) or \
- (metadata[baseurl]["timestamp"] < int(time.time()-(60*60*24)))):
+ (metadata[baseurl]["timestamp"] < int(time.time() - (60 * 60 * 24)))):
# Try to download new cache until we succeed on one.
- data=""
- for trynum in [1,2,3]:
+ data = ""
+ for trynum in [1, 2, 3]:
mytempfile = tempfile.TemporaryFile()
try:
- file_get(baseurl+"/"+mfile, mytempfile, conn)
+ file_get(baseurl + "/" + mfile, mytempfile, conn)
if mytempfile.tell() > len(data):
mytempfile.seek(0)
data = mytempfile.read()
except ValueError as e:
- sys.stderr.write("--- "+str(e)+"\n")
+ sys.stderr.write("--- %s\n" % str(e))
if trynum < 3:
sys.stderr.write(_("Retrying...\n"))
sys.stderr.flush()
mytempfile.close()
continue
- if match_in_array([mfile],suffix=".gz"):
+ if match_in_array([mfile], suffix=".gz"):
out.write("gzip'd\n")
out.flush()
try:
import gzip
mytempfile.seek(0)
- gzindex = gzip.GzipFile(mfile[:-3],'rb',9,mytempfile)
+ gzindex = gzip.GzipFile(mfile[:-3], 'rb', 9, mytempfile)
data = gzindex.read()
except SystemExit as e:
raise
except Exception as e:
mytempfile.close()
- sys.stderr.write(_("!!! Failed to use gzip: ")+str(e)+"\n")
+ sys.stderr.write(_("!!! Failed to use gzip: ") + str(e) + "\n")
sys.stderr.flush()
mytempfile.close()
try:
@@ -638,8 +698,8 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
except SystemExit as e:
raise
except Exception as e:
- sys.stderr.write(_("!!! Failed to read data from index: ")+str(mfile)+"\n")
- sys.stderr.write("!!! "+str(e)+"\n")
+ sys.stderr.write(_("!!! Failed to read data from index: ") + str(mfile) + "\n")
+ sys.stderr.write("!!! %s" % str(e))
sys.stderr.flush()
try:
metadatafile = open(_unicode_encode(metadatafilename,
@@ -650,7 +710,7 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
raise
except Exception as e:
sys.stderr.write(_("!!! Failed to write binary metadata to disk!\n"))
- sys.stderr.write("!!! "+str(e)+"\n")
+ sys.stderr.write("!!! %s\n" % str(e))
sys.stderr.flush()
break
# We may have metadata... now we run through the tbz2 list and check.
@@ -670,8 +730,8 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
self.display()
def display(self):
self.out.write("\r"+colorize("WARN",
- _("cache miss: '")+str(self.misses)+"'") + \
- " --- "+colorize("GOOD", _("cache hit: '")+str(self.hits)+"'"))
+ _("cache miss: '") + str(self.misses) + "'") + \
+ " --- " + colorize("GOOD", _("cache hit: '") + str(self.hits) + "'"))
self.out.flush()
cache_stats = CacheStats(out)
@@ -688,7 +748,7 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
cache_stats.update()
metadata[baseurl]["modified"] = 1
myid = None
- for retry in range(3):
+ for _x in range(3):
try:
myid = file_get_metadata(
"/".join((baseurl.rstrip("/"), x.lstrip("/"))),
@@ -699,22 +759,20 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache=
# make_http_request(). The docstring for this error in
# httplib.py says "Presumably, the server closed the
# connection before sending a valid response".
- conn, protocol, address, params, headers = create_conn(
- baseurl)
+ conn = create_conn(baseurl)[0]
except http_client_ResponseNotReady:
# With some http servers this error is known to be thrown
# from conn.getresponse() in make_http_request() when the
# remote file does not have appropriate read permissions.
# Maybe it's possible to recover from this exception in
# cases though, so retry.
- conn, protocol, address, params, headers = create_conn(
- baseurl)
+ conn = create_conn(baseurl)[0]
if myid and myid[0]:
metadata[baseurl]["data"][x] = make_metadata_dict(myid)
elif verbose:
sys.stderr.write(colorize("BAD",
- _("!!! Failed to retrieve metadata on: "))+str(x)+"\n")
+ _("!!! Failed to retrieve metadata on: ")) + str(x) + "\n")
sys.stderr.flush()
else:
cache_stats.hits += 1
@@ -861,7 +919,6 @@ class PackageIndex(object):
for metadata in sorted(self.packages,
key=portage.util.cmp_sort_key(_cmp_cpv)):
metadata = metadata.copy()
- cpv = metadata["CPV"]
if self._inherited_keys:
for k in self._inherited_keys:
v = self.header.get(k)
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.