argo_navis: venv/lib/python2.7/site-packages/boto/connection.py comparison

comparison venv/lib/python2.7/site-packages/boto/connection.py @ 0:d67268158946 draft

planemo upload commit a3f181f5f126803c654b3a66dd4e83a48f7e203b

author	bcclaywell
date	Mon, 12 Oct 2015 17:43:33 -0400
parents
children

comparison

equal deleted inserted replaced

--1:000000000000
+:d67268158946
+# Copyright (c) 2006-2012 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2012 Amazon.com, Inc. or its affiliates.
+# Copyright (c) 2010 Google
+# Copyright (c) 2008 rPath, Inc.
+# Copyright (c) 2009 The Echo Nest Corporation
+# Copyright (c) 2010, Eucalyptus Systems, Inc.
+# Copyright (c) 2011, Nexenta Systems Inc.
+# All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+# Parts of this code were copied or derived from sample code supplied by AWS.
+# The following notice applies to that code.
+#
+#  This software code is made available "AS IS" without warranties of any
+#  kind.  You may copy, display, modify and redistribute the software
+#  code either by itself or as incorporated into your code; provided that
+#  you do not remove any proprietary notices.  Your use of this software
+#  code is at your own risk and you waive any claim against Amazon
+#  Digital Services, Inc. or its affiliates with respect to your use of
+#  this software code. (c) 2006 Amazon Digital Services, Inc. or its
+#  affiliates.
+"""
+Handles basic connections to AWS
+"""
+from datetime import datetime
+import errno
+import os
+import random
+import re
+import socket
+import sys
+import time
+import xml.sax
+import copy
+from boto import auth
+from boto import auth_handler
+import boto
+import boto.utils
+import boto.handler
+import boto.cacerts
+from boto import config, UserAgent
+from boto.compat import six, http_client, urlparse, quote, encodebytes
+from boto.exception import AWSConnectionError
+from boto.exception import BotoClientError
+from boto.exception import BotoServerError
+from boto.exception import PleaseRetryException
+from boto.provider import Provider
+from boto.resultset import ResultSet
+HAVE_HTTPS_CONNECTION = False
+try:
+import ssl
+from boto import https_connection
+# Google App Engine runs on Python 2.5 so doesn't have ssl.SSLError.
+if hasattr(ssl, 'SSLError'):
+HAVE_HTTPS_CONNECTION = True
+except ImportError:
+pass
+try:
+import threading
+except ImportError:
+import dummy_threading as threading
+ON_APP_ENGINE = all(key in os.environ for key in (
+'USER_IS_ADMIN', 'CURRENT_VERSION_ID', 'APPLICATION_ID'))
+PORTS_BY_SECURITY = {True: 443,
+False: 80}
+DEFAULT_CA_CERTS_FILE = os.path.join(os.path.dirname(os.path.abspath(boto.cacerts.__file__)), "cacerts.txt")
+class HostConnectionPool(object):
+"""
+A pool of connections for one remote (host,port,is_secure).
+When connections are added to the pool, they are put into a
+pending queue.  The _mexe method returns connections to the pool
+before the response body has been read, so they connections aren't
+ready to send another request yet.  They stay in the pending queue
+until they are ready for another request, at which point they are
+returned to the pool of ready connections.
+The pool of ready connections is an ordered list of
+(connection,time) pairs, where the time is the time the connection
+was returned from _mexe.  After a certain period of time,
+connections are considered stale, and discarded rather than being
+reused.  This saves having to wait for the connection to time out
+if AWS has decided to close it on the other end because of
+inactivity.
+Thread Safety:
+This class is used only from ConnectionPool while it's mutex
+is held.
+"""
+def __init__(self):
+self.queue = []
+def size(self):
+"""
+Returns the number of connections in the pool for this host.
+Some of the connections may still be in use, and may not be
+ready to be returned by get().
+"""
+return len(self.queue)
+def put(self, conn):
+"""
+Adds a connection to the pool, along with the time it was
+added.
+"""
+self.queue.append((conn, time.time()))
+def get(self):
+"""
+Returns the next connection in this pool that is ready to be
+reused.  Returns None if there aren't any.
+"""
+# Discard ready connections that are too old.
+self.clean()
+# Return the first connection that is ready, and remove it
+# from the queue.  Connections that aren't ready are returned
+# to the end of the queue with an updated time, on the
+# assumption that somebody is actively reading the response.
+for _ in range(len(self.queue)):
+(conn, _) = self.queue.pop(0)
+if self._conn_ready(conn):
+return conn
+else:
+self.put(conn)
+return None
+def _conn_ready(self, conn):
+"""
+There is a nice state diagram at the top of http_client.py.  It
+indicates that once the response headers have been read (which
+_mexe does before adding the connection to the pool), a
+response is attached to the connection, and it stays there
+until it's done reading.  This isn't entirely true: even after
+the client is done reading, the response may be closed, but
+not removed from the connection yet.
+This is ugly, reading a private instance variable, but the
+state we care about isn't available in any public methods.
+"""
+if ON_APP_ENGINE:
+# Google AppEngine implementation of HTTPConnection doesn't contain
+# _HTTPConnection__response attribute. Moreover, it's not possible
+# to determine if given connection is ready. Reusing connections
+# simply doesn't make sense with App Engine urlfetch service.
+return False
+else:
+response = getattr(conn, '_HTTPConnection__response', None)
+return (response is None) or response.isclosed()
+def clean(self):
+"""
+Get rid of stale connections.
+"""
+# Note that we do not close the connection here -- somebody
+# may still be reading from it.
+while len(self.queue) > 0 and self._pair_stale(self.queue[0]):
+self.queue.pop(0)
+def _pair_stale(self, pair):
+"""
+Returns true of the (connection,time) pair is too old to be
+used.
+"""
+(_conn, return_time) = pair
+now = time.time()
+return return_time + ConnectionPool.STALE_DURATION < now
+class ConnectionPool(object):
+"""
+A connection pool that expires connections after a fixed period of
+time.  This saves time spent waiting for a connection that AWS has
+timed out on the other end.
+This class is thread-safe.
+"""
+#
+# The amout of time between calls to clean.
+#
+CLEAN_INTERVAL = 5.0
+#
+# How long before a connection becomes "stale" and won't be reused
+# again.  The intention is that this time is less that the timeout
+# period that AWS uses, so we'll never try to reuse a connection
+# and find that AWS is timing it out.
+#
+# Experimentation in July 2011 shows that AWS starts timing things
+# out after three minutes.  The 60 seconds here is conservative so
+# we should never hit that 3-minute timout.
+#
+STALE_DURATION = 60.0
+def __init__(self):
+# Mapping from (host,port,is_secure) to HostConnectionPool.
+# If a pool becomes empty, it is removed.
+self.host_to_pool = {}
+# The last time the pool was cleaned.
+self.last_clean_time = 0.0
+self.mutex = threading.Lock()
+ConnectionPool.STALE_DURATION = \
+config.getfloat('Boto', 'connection_stale_duration',
+ConnectionPool.STALE_DURATION)
+def __getstate__(self):
+pickled_dict = copy.copy(self.__dict__)
+pickled_dict['host_to_pool'] = {}
+del pickled_dict['mutex']
+return pickled_dict
+def __setstate__(self, dct):
+self.__init__()
+def size(self):
+"""
+Returns the number of connections in the pool.
+"""
+return sum(pool.size() for pool in self.host_to_pool.values())
+def get_http_connection(self, host, port, is_secure):
+"""
+Gets a connection from the pool for the named host.  Returns
+None if there is no connection that can be reused. It's the caller's
+responsibility to call close() on the connection when it's no longer
+needed.
+"""
+self.clean()
+with self.mutex:
+key = (host, port, is_secure)
+if key not in self.host_to_pool:
+return None
+return self.host_to_pool[key].get()
+def put_http_connection(self, host, port, is_secure, conn):
+"""
+Adds a connection to the pool of connections that can be
+reused for the named host.
+"""
+with self.mutex:
+key = (host, port, is_secure)
+if key not in self.host_to_pool:
+self.host_to_pool[key] = HostConnectionPool()
+self.host_to_pool[key].put(conn)
+def clean(self):
+"""
+Clean up the stale connections in all of the pools, and then
+get rid of empty pools.  Pools clean themselves every time a
+connection is fetched; this cleaning takes care of pools that
+aren't being used any more, so nothing is being gotten from
+them.
+"""
+with self.mutex:
+now = time.time()
+if self.last_clean_time + self.CLEAN_INTERVAL < now:
+to_remove = []
+for (host, pool) in self.host_to_pool.items():
+pool.clean()
+if pool.size() == 0:
+to_remove.append(host)
+for host in to_remove:
+del self.host_to_pool[host]
+self.last_clean_time = now
+class HTTPRequest(object):
+def __init__(self, method, protocol, host, port, path, auth_path,
+params, headers, body):
+"""Represents an HTTP request.
+:type method: string
+:param method: The HTTP method name, 'GET', 'POST', 'PUT' etc.
+:type protocol: string
+:param protocol: The http protocol used, 'http' or 'https'.
+:type host: string
+:param host: Host to which the request is addressed. eg. abc.com
+:type port: int
+:param port: port on which the request is being sent. Zero means unset,
+in which case default port will be chosen.
+:type path: string
+:param path: URL path that is being accessed.
+:type auth_path: string
+:param path: The part of the URL path used when creating the
+authentication string.
+:type params: dict
+:param params: HTTP url query parameters, with key as name of
+the param, and value as value of param.
+:type headers: dict
+:param headers: HTTP headers, with key as name of the header and value
+as value of header.
+:type body: string
+:param body: Body of the HTTP request. If not present, will be None or
+empty string ('').
+"""
+self.method = method
+self.protocol = protocol
+self.host = host
+self.port = port
+self.path = path
+if auth_path is None:
+auth_path = path
+self.auth_path = auth_path
+self.params = params
+# chunked Transfer-Encoding should act only on PUT request.
+if headers and 'Transfer-Encoding' in headers and \
+headers['Transfer-Encoding'] == 'chunked' and \
+self.method != 'PUT':
+self.headers = headers.copy()
+del self.headers['Transfer-Encoding']
+else:
+self.headers = headers
+self.body = body
+def __str__(self):
+return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) '
+'params(%s) headers(%s) body(%s)') % (self.method,
+self.protocol, self.host, self.port, self.path, self.params,
+self.headers, self.body))
+def authorize(self, connection, **kwargs):
+if not getattr(self, '_headers_quoted', False):
+for key in self.headers:
+val = self.headers[key]
+if isinstance(val, six.text_type):
+safe = '!"#$%&\'()*+,/:;<=>?@[\\]^`{|}~'
+self.headers[key] = quote(val.encode('utf-8'), safe)
+setattr(self, '_headers_quoted', True)
+self.headers['User-Agent'] = UserAgent
+connection._auth_handler.add_auth(self, **kwargs)
+# I'm not sure if this is still needed, now that add_auth is
+# setting the content-length for POST requests.
+if 'Content-Length' not in self.headers:
+if 'Transfer-Encoding' not in self.headers or \
+self.headers['Transfer-Encoding'] != 'chunked':
+self.headers['Content-Length'] = str(len(self.body))
+class HTTPResponse(http_client.HTTPResponse):
+def __init__(self, *args, **kwargs):
+http_client.HTTPResponse.__init__(self, *args, **kwargs)
+self._cached_response = ''
+def read(self, amt=None):
+"""Read the response.
+This method does not have the same behavior as
+http_client.HTTPResponse.read.  Instead, if this method is called with
+no ``amt`` arg, then the response body will be cached.  Subsequent
+calls to ``read()`` with no args **will return the cached response**.
+"""
+if amt is None:
+# The reason for doing this is that many places in boto call
+# response.read() and except to get the response body that they
+# can then process.  To make sure this always works as they expect
+# we're caching the response so that multiple calls to read()
+# will return the full body.  Note that this behavior only
+# happens if the amt arg is not specified.
+if not self._cached_response:
+self._cached_response = http_client.HTTPResponse.read(self)
+return self._cached_response
+else:
+return http_client.HTTPResponse.read(self, amt)
+class AWSAuthConnection(object):
+def __init__(self, host, aws_access_key_id=None,
+aws_secret_access_key=None,
+is_secure=True, port=None, proxy=None, proxy_port=None,
+proxy_user=None, proxy_pass=None, debug=0,
+https_connection_factory=None, path='/',
+provider='aws', security_token=None,
+suppress_consec_slashes=True,
+validate_certs=True, profile_name=None):
+"""
+:type host: str
+:param host: The host to make the connection to
+:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
+Amazon). If none is specified, the value in your
+``AWS_ACCESS_KEY_ID`` environmental variable is used.
+:keyword str aws_secret_access_key: Your AWS Secret Access Key
+(provided by Amazon). If none is specified, the value in your
+``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
+:keyword str security_token: The security token associated with
+temporary credentials issued by STS.  Optional unless using
+temporary credentials.  If none is specified, the environment
+variable ``AWS_SECURITY_TOKEN`` is used if defined.
+:type is_secure: boolean
+:param is_secure: Whether the connection is over SSL
+:type https_connection_factory: list or tuple
+:param https_connection_factory: A pair of an HTTP connection
+factory and the exceptions to catch.  The factory should have
+a similar interface to L{http_client.HTTPSConnection}.
+:param str proxy: Address/hostname for a proxy server
+:type proxy_port: int
+:param proxy_port: The port to use when connecting over a proxy
+:type proxy_user: str
+:param proxy_user: The username to connect with on the proxy
+:type proxy_pass: str
+:param proxy_pass: The password to use when connection over a proxy.
+:type port: int
+:param port: The port to use to connect
+:type suppress_consec_slashes: bool
+:param suppress_consec_slashes: If provided, controls whether
+consecutive slashes will be suppressed in key paths.
+:type validate_certs: bool
+:param validate_certs: Controls whether SSL certificates
+will be validated or not.  Defaults to True.
+:type profile_name: str
+:param profile_name: Override usual Credentials section in config
+file to use a named set of keys instead.
+"""
+self.suppress_consec_slashes = suppress_consec_slashes
+self.num_retries = 6
+# Override passed-in is_secure setting if value was defined in config.
+if config.has_option('Boto', 'is_secure'):
+is_secure = config.getboolean('Boto', 'is_secure')
+self.is_secure = is_secure
+# Whether or not to validate server certificates.
+# The default is now to validate certificates.  This can be
+# overridden in the boto config file are by passing an
+# explicit validate_certs parameter to the class constructor.
+self.https_validate_certificates = config.getbool(
+'Boto', 'https_validate_certificates',
+validate_certs)
+if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
+raise BotoClientError(
+"SSL server certificate validation is enabled in boto "
+"configuration, but Python dependencies required to "
+"support this feature are not available. Certificate "
+"validation is only supported when running under Python "
+"2.6 or later.")
+certs_file = config.get_value(
+'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
+if certs_file == 'system':
+certs_file = None
+self.ca_certificates_file = certs_file
+if port:
+self.port = port
+else:
+self.port = PORTS_BY_SECURITY[is_secure]
+self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
+# define exceptions from http_client that we want to catch and retry
+self.http_exceptions = (http_client.HTTPException, socket.error,
+socket.gaierror, http_client.BadStatusLine)
+# define subclasses of the above that are not retryable.
+self.http_unretryable_exceptions = []
+if HAVE_HTTPS_CONNECTION:
+self.http_unretryable_exceptions.append(
+https_connection.InvalidCertificateException)
+# define values in socket exceptions we don't want to catch
+self.socket_exception_values = (errno.EINTR,)
+if https_connection_factory is not None:
+self.https_connection_factory = https_connection_factory[0]
+self.http_exceptions += https_connection_factory[1]
+else:
+self.https_connection_factory = None
+if (is_secure):
+self.protocol = 'https'
+else:
+self.protocol = 'http'
+self.host = host
+self.path = path
+# if the value passed in for debug
+if not isinstance(debug, six.integer_types):
+debug = 0
+self.debug = config.getint('Boto', 'debug', debug)
+self.host_header = None
+# Timeout used to tell http_client how long to wait for socket timeouts.
+# Default is to leave timeout unchanged, which will in turn result in
+# the socket's default global timeout being used. To specify a
+# timeout, set http_socket_timeout in Boto config. Regardless,
+# timeouts will only be applied if Python is 2.6 or greater.
+self.http_connection_kwargs = {}
+if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
+# If timeout isn't defined in boto config file, use 70 second
+# default as recommended by
+# http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html
+self.http_connection_kwargs['timeout'] = config.getint(
+'Boto', 'http_socket_timeout', 70)
+if isinstance(provider, Provider):
+# Allow overriding Provider
+self.provider = provider
+else:
+self._provider_type = provider
+self.provider = Provider(self._provider_type,
+aws_access_key_id,
+aws_secret_access_key,
+security_token,
+profile_name)
+# Allow config file to override default host, port, and host header.
+if self.provider.host:
+self.host = self.provider.host
+if self.provider.port:
+self.port = self.provider.port
+if self.provider.host_header:
+self.host_header = self.provider.host_header
+self._pool = ConnectionPool()
+self._connection = (self.host, self.port, self.is_secure)
+self._last_rs = None
+self._auth_handler = auth.get_auth_handler(
+host, config, self.provider, self._required_auth_capability())
+if getattr(self, 'AuthServiceName', None) is not None:
+self.auth_service_name = self.AuthServiceName
+self.request_hook = None
+def __repr__(self):
+return '%s:%s' % (self.__class__.__name__, self.host)
+def _required_auth_capability(self):
+return []
+def _get_auth_service_name(self):
+return getattr(self._auth_handler, 'service_name')
+# For Sigv4, the auth_service_name/auth_region_name properties allow
+# the service_name/region_name to be explicitly set instead of being
+# derived from the endpoint url.
+def _set_auth_service_name(self, value):
+self._auth_handler.service_name = value
+auth_service_name = property(_get_auth_service_name, _set_auth_service_name)
+def _get_auth_region_name(self):
+return getattr(self._auth_handler, 'region_name')
+def _set_auth_region_name(self, value):
+self._auth_handler.region_name = value
+auth_region_name = property(_get_auth_region_name, _set_auth_region_name)
+def connection(self):
+return self.get_http_connection(*self._connection)
+connection = property(connection)
+def aws_access_key_id(self):
+return self.provider.access_key
+aws_access_key_id = property(aws_access_key_id)
+gs_access_key_id = aws_access_key_id
+access_key = aws_access_key_id
+def aws_secret_access_key(self):
+return self.provider.secret_key
+aws_secret_access_key = property(aws_secret_access_key)
+gs_secret_access_key = aws_secret_access_key
+secret_key = aws_secret_access_key
+def profile_name(self):
+return self.provider.profile_name
+profile_name = property(profile_name)
+def get_path(self, path='/'):
+# The default behavior is to suppress consecutive slashes for reasons
+# discussed at
+# https://groups.google.com/forum/#!topic/boto-dev/-ft0XPUy0y8
+# You can override that behavior with the suppress_consec_slashes param.
+if not self.suppress_consec_slashes:
+return self.path + re.sub('^(/*)/', "\\1", path)
+pos = path.find('?')
+if pos >= 0:
+params = path[pos:]
+path = path[:pos]
+else:
+params = None
+if path[-1] == '/':
+need_trailing = True
+else:
+need_trailing = False
+path_elements = self.path.split('/')
+path_elements.extend(path.split('/'))
+path_elements = [p for p in path_elements if p]
+path = '/' + '/'.join(path_elements)
+if path[-1] != '/' and need_trailing:
+path += '/'
+if params:
+path = path + params
+return path
+def server_name(self, port=None):
+if not port:
+port = self.port
+if port == 80:
+signature_host = self.host
+else:
+# This unfortunate little hack can be attributed to
+# a difference in the 2.6 version of http_client.  In old
+# versions, it would append ":443" to the hostname sent
+# in the Host header and so we needed to make sure we
+# did the same when calculating the V2 signature.  In 2.6
+# (and higher!)
+# it no longer does that.  Hence, this kludge.
+if ((ON_APP_ENGINE and sys.version[:3] == '2.5') or
+sys.version[:3] in ('2.6', '2.7')) and port == 443:
+signature_host = self.host
+else:
+signature_host = '%s:%d' % (self.host, port)
+return signature_host
+def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass):
+self.proxy = proxy
+self.proxy_port = proxy_port
+self.proxy_user = proxy_user
+self.proxy_pass = proxy_pass
+if 'http_proxy' in os.environ and not self.proxy:
+pattern = re.compile(
+'(?:http://)?'
+'(?:(?P<user>[\w\-\.]+):(?P<pass>.*)@)?'
+'(?P<host>[\w\-\.]+)'
+'(?::(?P<port>\d+))?'
+)
+match = pattern.match(os.environ['http_proxy'])
+if match:
+self.proxy = match.group('host')
+self.proxy_port = match.group('port')
+self.proxy_user = match.group('user')
+self.proxy_pass = match.group('pass')
+else:
+if not self.proxy:
+self.proxy = config.get_value('Boto', 'proxy', None)
+if not self.proxy_port:
+self.proxy_port = config.get_value('Boto', 'proxy_port', None)
+if not self.proxy_user:
+self.proxy_user = config.get_value('Boto', 'proxy_user', None)
+if not self.proxy_pass:
+self.proxy_pass = config.get_value('Boto', 'proxy_pass', None)
+if not self.proxy_port and self.proxy:
+print("http_proxy environment variable does not specify "
+"a port, using default")
+self.proxy_port = self.port
+self.no_proxy = os.environ.get('no_proxy', '') or os.environ.get('NO_PROXY', '')
+self.use_proxy = (self.proxy is not None)
+def get_http_connection(self, host, port, is_secure):
+conn = self._pool.get_http_connection(host, port, is_secure)
+if conn is not None:
+return conn
+else:
+return self.new_http_connection(host, port, is_secure)
+def skip_proxy(self, host):
+if not self.no_proxy:
+return False
+if self.no_proxy == "*":
+return True
+hostonly = host
+hostonly = host.split(':')[0]
+for name in self.no_proxy.split(','):
+if name and (hostonly.endswith(name) or host.endswith(name)):
+return True
+return False
+def new_http_connection(self, host, port, is_secure):
+if host is None:
+host = self.server_name()
+# Make sure the host is really just the host, not including
+# the port number
+host = host.split(':', 1)[0]
+http_connection_kwargs = self.http_connection_kwargs.copy()
+# Connection factories below expect a port keyword argument
+http_connection_kwargs['port'] = port
+# Override host with proxy settings if needed
+if self.use_proxy and not is_secure and \
+not self.skip_proxy(host):
+host = self.proxy
+http_connection_kwargs['port'] = int(self.proxy_port)
+if is_secure:
+boto.log.debug(
+'establishing HTTPS connection: host=%s, kwargs=%s',
+host, http_connection_kwargs)
+if self.use_proxy and not self.skip_proxy(host):
+connection = self.proxy_ssl(host, is_secure and 443 or 80)
+elif self.https_connection_factory:
+connection = self.https_connection_factory(host)
+elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
+connection = https_connection.CertValidatingHTTPSConnection(
+host, ca_certs=self.ca_certificates_file,
+**http_connection_kwargs)
+else:
+connection = http_client.HTTPSConnection(
+host, **http_connection_kwargs)
+else:
+boto.log.debug('establishing HTTP connection: kwargs=%s' %
+http_connection_kwargs)
+if self.https_connection_factory:
+# even though the factory says https, this is too handy
+# to not be able to allow overriding for http also.
+connection = self.https_connection_factory(
+host, **http_connection_kwargs)
+else:
+connection = http_client.HTTPConnection(
+host, **http_connection_kwargs)
+if self.debug > 1:
+connection.set_debuglevel(self.debug)
+# self.connection must be maintained for backwards-compatibility
+# however, it must be dynamically pulled from the connection pool
+# set a private variable which will enable that
+if host.split(':')[0] == self.host and is_secure == self.is_secure:
+self._connection = (host, port, is_secure)
+# Set the response class of the http connection to use our custom
+# class.
+connection.response_class = HTTPResponse
+return connection
+def put_http_connection(self, host, port, is_secure, connection):
+self._pool.put_http_connection(host, port, is_secure, connection)
+def proxy_ssl(self, host=None, port=None):
+if host and port:
+host = '%s:%d' % (host, port)
+else:
+host = '%s:%d' % (self.host, self.port)
+# Seems properly to use timeout for connect too
+timeout = self.http_connection_kwargs.get("timeout")
+if timeout is not None:
+sock = socket.create_connection((self.proxy,
+int(self.proxy_port)), timeout)
+else:
+sock = socket.create_connection((self.proxy, int(self.proxy_port)))
+boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host)
+sock.sendall("CONNECT %s HTTP/1.0\r\n" % host)
+sock.sendall("User-Agent: %s\r\n" % UserAgent)
+if self.proxy_user and self.proxy_pass:
+for k, v in self.get_proxy_auth_header().items():
+sock.sendall("%s: %s\r\n" % (k, v))
+# See discussion about this config option at
+# https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc
+if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False):
+sock.sendall("\r\n")
+else:
+sock.sendall("\r\n")
+resp = http_client.HTTPResponse(sock, strict=True, debuglevel=self.debug)
+resp.begin()
+if resp.status != 200:
+# Fake a socket error, use a code that make it obvious it hasn't
+# been generated by the socket library
+raise socket.error(-71,
+"Error talking to HTTP proxy %s:%s: %s (%s)" %
+(self.proxy, self.proxy_port,
+resp.status, resp.reason))
+# We can safely close the response, it duped the original socket
+resp.close()
+h = http_client.HTTPConnection(host)
+if self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
+msg = "wrapping ssl socket for proxied connection; "
+if self.ca_certificates_file:
+msg += "CA certificate file=%s" % self.ca_certificates_file
+else:
+msg += "using system provided SSL certs"
+boto.log.debug(msg)
+key_file = self.http_connection_kwargs.get('key_file', None)
+cert_file = self.http_connection_kwargs.get('cert_file', None)
+sslSock = ssl.wrap_socket(sock, keyfile=key_file,
+certfile=cert_file,
+cert_reqs=ssl.CERT_REQUIRED,
+ca_certs=self.ca_certificates_file)
+cert = sslSock.getpeercert()
+hostname = self.host.split(':', 0)[0]
+if not https_connection.ValidateCertificateHostname(cert, hostname):
+raise https_connection.InvalidCertificateException(
+hostname, cert, 'hostname mismatch')
+else:
+# Fallback for old Python without ssl.wrap_socket
+if hasattr(http_client, 'ssl'):
+sslSock = http_client.ssl.SSLSocket(sock)
+else:
+sslSock = socket.ssl(sock, None, None)
+sslSock = http_client.FakeSocket(sock, sslSock)
+# This is a bit unclean
+h.sock = sslSock
+return h
+def prefix_proxy_to_path(self, path, host=None):
+path = self.protocol + '://' + (host or self.server_name()) + path
+return path
+def get_proxy_auth_header(self):
+auth = encodebytes(self.proxy_user + ':' + self.proxy_pass)
+return {'Proxy-Authorization': 'Basic %s' % auth}
+# For passing proxy information to other connection libraries, e.g. cloudsearch2
+def get_proxy_url_with_auth(self):
+if not self.use_proxy:
+return None
+if self.proxy_user or self.proxy_pass:
+if self.proxy_pass:
+login_info = '%s:%s@' % (self.proxy_user, self.proxy_pass)
+else:
+login_info = '%s@' % self.proxy_user
+else:
+login_info = ''
+return 'http://%s%s:%s' % (login_info, self.proxy, str(self.proxy_port or self.port))
+def set_host_header(self, request):
+try:
+request.headers['Host'] = \
+self._auth_handler.host_header(self.host, request)
+except AttributeError:
+request.headers['Host'] = self.host.split(':', 1)[0]
+def set_request_hook(self, hook):
+self.request_hook = hook
+def _mexe(self, request, sender=None, override_num_retries=None,
+retry_handler=None):
+"""
+mexe - Multi-execute inside a loop, retrying multiple times to handle
+transient Internet errors by simply trying again.
+Also handles redirects.
+This code was inspired by the S3Utils classes posted to the boto-users
+Google group by Larry Bates.  Thanks!
+"""
+boto.log.debug('Method: %s' % request.method)
+boto.log.debug('Path: %s' % request.path)
+boto.log.debug('Data: %s' % request.body)
+boto.log.debug('Headers: %s' % request.headers)
+boto.log.debug('Host: %s' % request.host)
+boto.log.debug('Port: %s' % request.port)
+boto.log.debug('Params: %s' % request.params)
+response = None
+body = None
+ex = None
+if override_num_retries is None:
+num_retries = config.getint('Boto', 'num_retries', self.num_retries)
+else:
+num_retries = override_num_retries
+i = 0
+connection = self.get_http_connection(request.host, request.port,
+self.is_secure)
+# Convert body to bytes if needed
+if not isinstance(request.body, bytes) and hasattr(request.body,
+'encode'):
+request.body = request.body.encode('utf-8')
+while i <= num_retries:
+# Use binary exponential backoff to desynchronize client requests.
+next_sleep = min(random.random() * (2 ** i),
+boto.config.get('Boto', 'max_retry_delay', 60))
+try:
+# we now re-sign each request before it is retried
+boto.log.debug('Token: %s' % self.provider.security_token)
+request.authorize(connection=self)
+# Only force header for non-s3 connections, because s3 uses
+# an older signing method + bucket resource URLs that include
+# the port info. All others should be now be up to date and
+# not include the port.
+if 's3' not in self._required_auth_capability():
+if not getattr(self, 'anon', False):
+if not request.headers.get('Host'):
+self.set_host_header(request)
+boto.log.debug('Final headers: %s' % request.headers)
+request.start_time = datetime.now()
+if callable(sender):
+response = sender(connection, request.method, request.path,
+request.body, request.headers)
+else:
+connection.request(request.method, request.path,
+request.body, request.headers)
+response = connection.getresponse()
+boto.log.debug('Response headers: %s' % response.getheaders())
+location = response.getheader('location')
+# -- gross hack --
+# http_client gets confused with chunked responses to HEAD requests
+# so I have to fake it out
+if request.method == 'HEAD' and getattr(response,
+'chunked', False):
+response.chunked = 0
+if callable(retry_handler):
+status = retry_handler(response, i, next_sleep)
+if status:
+msg, i, next_sleep = status
+if msg:
+boto.log.debug(msg)
+time.sleep(next_sleep)
+continue
+if response.status in [500, 502, 503, 504]:
+msg = 'Received %d response.  ' % response.status
+msg += 'Retrying in %3.1f seconds' % next_sleep
+boto.log.debug(msg)
+body = response.read()
+if isinstance(body, bytes):
+body = body.decode('utf-8')
+elif response.status < 300 or response.status >= 400 or \
+not location:
+# don't return connection to the pool if response contains
+# Connection:close header, because the connection has been
+# closed and default reconnect behavior may do something
+# different than new_http_connection. Also, it's probably
+# less efficient to try to reuse a closed connection.
+conn_header_value = response.getheader('connection')
+if conn_header_value == 'close':
+connection.close()
+else:
+self.put_http_connection(request.host, request.port,
+self.is_secure, connection)
+if self.request_hook is not None:
+self.request_hook.handle_request_data(request, response)
+return response
+else:
+scheme, request.host, request.path, \
+params, query, fragment = urlparse(location)
+if query:
+request.path += '?' + query
+# urlparse can return both host and port in netloc, so if
+# that's the case we need to split them up properly
+if ':' in request.host:
+request.host, request.port = request.host.split(':', 1)
+msg = 'Redirecting: %s' % scheme + '://'
+msg += request.host + request.path
+boto.log.debug(msg)
+connection = self.get_http_connection(request.host,
+request.port,
+scheme == 'https')
+response = None
+continue
+except PleaseRetryException as e:
+boto.log.debug('encountered a retry exception: %s' % e)
+connection = self.new_http_connection(request.host, request.port,
+self.is_secure)
+response = e.response
+ex = e
+except self.http_exceptions as e:
+for unretryable in self.http_unretryable_exceptions:
+if isinstance(e, unretryable):
+boto.log.debug(
+'encountered unretryable %s exception, re-raising' %
+e.__class__.__name__)
+raise
+boto.log.debug('encountered %s exception, reconnecting' %
+e.__class__.__name__)
+connection = self.new_http_connection(request.host, request.port,
+self.is_secure)
+ex = e
+time.sleep(next_sleep)
+i += 1
+# If we made it here, it's because we have exhausted our retries
+# and stil haven't succeeded.  So, if we have a response object,
+# use it to raise an exception.
+# Otherwise, raise the exception that must have already happened.
+if self.request_hook is not None:
+self.request_hook.handle_request_data(request, response, error=True)
+if response:
+raise BotoServerError(response.status, response.reason, body)
+elif ex:
+raise ex
+else:
+msg = 'Please report this exception as a Boto Issue!'
+raise BotoClientError(msg)
+def build_base_http_request(self, method, path, auth_path,
+params=None, headers=None, data='', host=None):
+path = self.get_path(path)
+if auth_path is not None:
+auth_path = self.get_path(auth_path)
+if params is None:
+params = {}
+else:
+params = params.copy()
+if headers is None:
+headers = {}
+else:
+headers = headers.copy()
+if self.host_header and not boto.utils.find_matching_headers('host', headers):
+headers['host'] = self.host_header
+host = host or self.host
+if self.use_proxy:
+if not auth_path:
+auth_path = path
+path = self.prefix_proxy_to_path(path, host)
+if self.proxy_user and self.proxy_pass and not self.is_secure:
+# If is_secure, we don't have to set the proxy authentication
+# header here, we did that in the CONNECT to the proxy.
+headers.update(self.get_proxy_auth_header())
+return HTTPRequest(method, self.protocol, host, self.port,
+path, auth_path, params, headers, data)
+def make_request(self, method, path, headers=None, data='', host=None,
+auth_path=None, sender=None, override_num_retries=None,
+params=None, retry_handler=None):
+"""Makes a request to the server, with stock multiple-retry logic."""
+if params is None:
+params = {}
+http_request = self.build_base_http_request(method, path, auth_path,
+params, headers, data, host)
+return self._mexe(http_request, sender, override_num_retries,
+retry_handler=retry_handler)
+def close(self):
+"""(Optional) Close any open HTTP connections.  This is non-destructive,
+and making a new request will open a connection again."""
+boto.log.debug('closing all HTTP connections')
+self._connection = None  # compat field
+class AWSQueryConnection(AWSAuthConnection):
+APIVersion = ''
+ResponseError = BotoServerError
+def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
+is_secure=True, port=None, proxy=None, proxy_port=None,
+proxy_user=None, proxy_pass=None, host=None, debug=0,
+https_connection_factory=None, path='/', security_token=None,
+validate_certs=True, profile_name=None, provider='aws'):
+super(AWSQueryConnection, self).__init__(
+host, aws_access_key_id,
+aws_secret_access_key,
+is_secure, port, proxy,
+proxy_port, proxy_user, proxy_pass,
+debug, https_connection_factory, path,
+security_token=security_token,
+validate_certs=validate_certs,
+profile_name=profile_name,
+provider=provider)
+def _required_auth_capability(self):
+return []
+def get_utf8_value(self, value):
+return boto.utils.get_utf8_value(value)
+def make_request(self, action, params=None, path='/', verb='GET'):
+http_request = self.build_base_http_request(verb, path, None,
+params, {}, '',
+self.host)
+if action:
+http_request.params['Action'] = action
+if self.APIVersion:
+http_request.params['Version'] = self.APIVersion
+return self._mexe(http_request)
+def build_list_params(self, params, items, label):
+if isinstance(items, six.string_types):
+items = [items]
+for i in range(1, len(items) + 1):
+params['%s.%d' % (label, i)] = items[i - 1]
+def build_complex_list_params(self, params, items, label, names):
+"""Serialize a list of structures.
+For example::
+items = [('foo', 'bar', 'baz'), ('foo2', 'bar2', 'baz2')]
+label = 'ParamName.member'
+names = ('One', 'Two', 'Three')
+self.build_complex_list_params(params, items, label, names)
+would result in the params dict being updated with these params::
+ParamName.member.1.One = foo
+ParamName.member.1.Two = bar
+ParamName.member.1.Three = baz
+ParamName.member.2.One = foo2
+ParamName.member.2.Two = bar2
+ParamName.member.2.Three = baz2
+:type params: dict
+:param params: The params dict.  The complex list params
+will be added to this dict.
+:type items: list of tuples
+:param items: The list to serialize.
+:type label: string
+:param label: The prefix to apply to the parameter.
+:type names: tuple of strings
+:param names: The names associated with each tuple element.
+"""
+for i, item in enumerate(items, 1):
+current_prefix = '%s.%s' % (label, i)
+for key, value in zip(names, item):
+full_key = '%s.%s' % (current_prefix, key)
+params[full_key] = value
+# generics
+def get_list(self, action, params, markers, path='/',
+parent=None, verb='GET'):
+if not parent:
+parent = self
+response = self.make_request(action, params, path, verb)
+body = response.read()
+boto.log.debug(body)
+if not body:
+boto.log.error('Null body %s' % body)
+raise self.ResponseError(response.status, response.reason, body)
+elif response.status == 200:
+rs = ResultSet(markers)
+h = boto.handler.XmlHandler(rs, parent)
+if isinstance(body, six.text_type):
+body = body.encode('utf-8')
+xml.sax.parseString(body, h)
+return rs
+else:
+boto.log.error('%s %s' % (response.status, response.reason))
+boto.log.error('%s' % body)
+raise self.ResponseError(response.status, response.reason, body)
+def get_object(self, action, params, cls, path='/',
+parent=None, verb='GET'):
+if not parent:
+parent = self
+response = self.make_request(action, params, path, verb)
+body = response.read()
+boto.log.debug(body)
+if not body:
+boto.log.error('Null body %s' % body)
+raise self.ResponseError(response.status, response.reason, body)
+elif response.status == 200:
+obj = cls(parent)
+h = boto.handler.XmlHandler(obj, parent)
+if isinstance(body, six.text_type):
+body = body.encode('utf-8')
+xml.sax.parseString(body, h)
+return obj
+else:
+boto.log.error('%s %s' % (response.status, response.reason))
+boto.log.error('%s' % body)
+raise self.ResponseError(response.status, response.reason, body)
+def get_status(self, action, params, path='/', parent=None, verb='GET'):
+if not parent:
+parent = self
+response = self.make_request(action, params, path, verb)
+body = response.read()
+boto.log.debug(body)
+if not body:
+boto.log.error('Null body %s' % body)
+raise self.ResponseError(response.status, response.reason, body)
+elif response.status == 200:
+rs = ResultSet()
+h = boto.handler.XmlHandler(rs, parent)
+xml.sax.parseString(body, h)
+return rs.status
+else:
+boto.log.error('%s %s' % (response.status, response.reason))
+boto.log.error('%s' % body)
+raise self.ResponseError(response.status, response.reason, body)

Mercurial > repos > bcclaywell > argo_navis

comparison venv/lib/python2.7/site-packages/boto/connection.py @ 0:d67268158946 draft