Mercurial > repos > bcclaywell > argo_navis

comparison venv/lib/python2.7/site-packages/boto/provider.py @ 0:d67268158946 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit a3f181f5f126803c654b3a66dd4e83a48f7e203b
author bcclaywell
date Mon, 12 Oct 2015 17:43:33 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:d67268158946
1 # Copyright (c) 2010 Mitch Garnaat http://garnaat.org/
2 # Copyright 2010 Google Inc.
3 # Copyright (c) 2010, Eucalyptus Systems, Inc.
4 # Copyright (c) 2011, Nexenta Systems Inc.
5 # All rights reserved.
6 #
7 # Permission is hereby granted, free of charge, to any person obtaining a
8 # copy of this software and associated documentation files (the
9 # "Software"), to deal in the Software without restriction, including
10 # without limitation the rights to use, copy, modify, merge, publish, dis-
11 # tribute, sublicense, and/or sell copies of the Software, and to permit
12 # persons to whom the Software is furnished to do so, subject to the fol-
13 # lowing conditions:
14 #
15 # The above copyright notice and this permission notice shall be included
16 # in all copies or substantial portions of the Software.
17 #
18 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
20 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
21 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
22 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 # IN THE SOFTWARE.
25 """
26 This class encapsulates the provider-specific header differences.
27 """
28
29 import os
30 from boto.compat import six
31 from datetime import datetime
32
33 import boto
34 from boto import config
35 from boto.compat import expanduser
36 from boto.pyami.config import Config
37 from boto.gs.acl import ACL
38 from boto.gs.acl import CannedACLStrings as CannedGSACLStrings
39 from boto.s3.acl import CannedACLStrings as CannedS3ACLStrings
40 from boto.s3.acl import Policy
41
42
43 HEADER_PREFIX_KEY = 'header_prefix'
44 METADATA_PREFIX_KEY = 'metadata_prefix'
45
46 AWS_HEADER_PREFIX = 'x-amz-'
47 GOOG_HEADER_PREFIX = 'x-goog-'
48
49 ACL_HEADER_KEY = 'acl-header'
50 AUTH_HEADER_KEY = 'auth-header'
51 COPY_SOURCE_HEADER_KEY = 'copy-source-header'
52 COPY_SOURCE_VERSION_ID_HEADER_KEY = 'copy-source-version-id-header'
53 COPY_SOURCE_RANGE_HEADER_KEY = 'copy-source-range-header'
54 DELETE_MARKER_HEADER_KEY = 'delete-marker-header'
55 DATE_HEADER_KEY = 'date-header'
56 METADATA_DIRECTIVE_HEADER_KEY = 'metadata-directive-header'
57 RESUMABLE_UPLOAD_HEADER_KEY = 'resumable-upload-header'
58 SECURITY_TOKEN_HEADER_KEY = 'security-token-header'
59 STORAGE_CLASS_HEADER_KEY = 'storage-class'
60 MFA_HEADER_KEY = 'mfa-header'
61 SERVER_SIDE_ENCRYPTION_KEY = 'server-side-encryption-header'
62 VERSION_ID_HEADER_KEY = 'version-id-header'
63 RESTORE_HEADER_KEY = 'restore-header'
64
65 STORAGE_COPY_ERROR = 'StorageCopyError'
66 STORAGE_CREATE_ERROR = 'StorageCreateError'
67 STORAGE_DATA_ERROR = 'StorageDataError'
68 STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError'
69 STORAGE_RESPONSE_ERROR = 'StorageResponseError'
70 NO_CREDENTIALS_PROVIDED = object()
71
72
73 class ProfileNotFoundError(ValueError):
74 pass
75
76
77 class Provider(object):
78
79 CredentialMap = {
80 'aws': ('aws_access_key_id', 'aws_secret_access_key',
81 'aws_security_token', 'aws_profile'),
82 'google': ('gs_access_key_id', 'gs_secret_access_key',
83 None, None),
84 }
85
86 AclClassMap = {
87 'aws': Policy,
88 'google': ACL
89 }
90
91 CannedAclsMap = {
92 'aws': CannedS3ACLStrings,
93 'google': CannedGSACLStrings
94 }
95
96 HostKeyMap = {
97 'aws': 's3',
98 'google': 'gs'
99 }
100
101 ChunkedTransferSupport = {
102 'aws': False,
103 'google': True
104 }
105
106 MetadataServiceSupport = {
107 'aws': True,
108 'google': False
109 }
110
111 # If you update this map please make sure to put "None" for the
112 # right-hand-side for any headers that don't apply to a provider, rather
113 # than simply leaving that header out (which would cause KeyErrors).
114 HeaderInfoMap = {
115 'aws': {
116 HEADER_PREFIX_KEY: AWS_HEADER_PREFIX,
117 METADATA_PREFIX_KEY: AWS_HEADER_PREFIX + 'meta-',
118 ACL_HEADER_KEY: AWS_HEADER_PREFIX + 'acl',
119 AUTH_HEADER_KEY: 'AWS',
120 COPY_SOURCE_HEADER_KEY: AWS_HEADER_PREFIX + 'copy-source',
121 COPY_SOURCE_VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX +
122 'copy-source-version-id',
123 COPY_SOURCE_RANGE_HEADER_KEY: AWS_HEADER_PREFIX +
124 'copy-source-range',
125 DATE_HEADER_KEY: AWS_HEADER_PREFIX + 'date',
126 DELETE_MARKER_HEADER_KEY: AWS_HEADER_PREFIX + 'delete-marker',
127 METADATA_DIRECTIVE_HEADER_KEY: AWS_HEADER_PREFIX +
128 'metadata-directive',
129 RESUMABLE_UPLOAD_HEADER_KEY: None,
130 SECURITY_TOKEN_HEADER_KEY: AWS_HEADER_PREFIX + 'security-token',
131 SERVER_SIDE_ENCRYPTION_KEY: AWS_HEADER_PREFIX +
132 'server-side-encryption',
133 VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX + 'version-id',
134 STORAGE_CLASS_HEADER_KEY: AWS_HEADER_PREFIX + 'storage-class',
135 MFA_HEADER_KEY: AWS_HEADER_PREFIX + 'mfa',
136 RESTORE_HEADER_KEY: AWS_HEADER_PREFIX + 'restore',
137 },
138 'google': {
139 HEADER_PREFIX_KEY: GOOG_HEADER_PREFIX,
140 METADATA_PREFIX_KEY: GOOG_HEADER_PREFIX + 'meta-',
141 ACL_HEADER_KEY: GOOG_HEADER_PREFIX + 'acl',
142 AUTH_HEADER_KEY: 'GOOG1',
143 COPY_SOURCE_HEADER_KEY: GOOG_HEADER_PREFIX + 'copy-source',
144 COPY_SOURCE_VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX +
145 'copy-source-version-id',
146 COPY_SOURCE_RANGE_HEADER_KEY: None,
147 DATE_HEADER_KEY: GOOG_HEADER_PREFIX + 'date',
148 DELETE_MARKER_HEADER_KEY: GOOG_HEADER_PREFIX + 'delete-marker',
149 METADATA_DIRECTIVE_HEADER_KEY: GOOG_HEADER_PREFIX +
150 'metadata-directive',
151 RESUMABLE_UPLOAD_HEADER_KEY: GOOG_HEADER_PREFIX + 'resumable',
152 SECURITY_TOKEN_HEADER_KEY: GOOG_HEADER_PREFIX + 'security-token',
153 SERVER_SIDE_ENCRYPTION_KEY: None,
154 # Note that this version header is not to be confused with
155 # the Google Cloud Storage 'x-goog-api-version' header.
156 VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX + 'version-id',
157 STORAGE_CLASS_HEADER_KEY: None,
158 MFA_HEADER_KEY: None,
159 RESTORE_HEADER_KEY: None,
160 }
161 }
162
163 ErrorMap = {
164 'aws': {
165 STORAGE_COPY_ERROR: boto.exception.S3CopyError,
166 STORAGE_CREATE_ERROR: boto.exception.S3CreateError,
167 STORAGE_DATA_ERROR: boto.exception.S3DataError,
168 STORAGE_PERMISSIONS_ERROR: boto.exception.S3PermissionsError,
169 STORAGE_RESPONSE_ERROR: boto.exception.S3ResponseError,
170 },
171 'google': {
172 STORAGE_COPY_ERROR: boto.exception.GSCopyError,
173 STORAGE_CREATE_ERROR: boto.exception.GSCreateError,
174 STORAGE_DATA_ERROR: boto.exception.GSDataError,
175 STORAGE_PERMISSIONS_ERROR: boto.exception.GSPermissionsError,
176 STORAGE_RESPONSE_ERROR: boto.exception.GSResponseError,
177 }
178 }
179
180 def __init__(self, name, access_key=None, secret_key=None,
181 security_token=None, profile_name=None):
182 self.host = None
183 self.port = None
184 self.host_header = None
185 self.access_key = access_key
186 self.secret_key = secret_key
187 self.security_token = security_token
188 self.profile_name = profile_name
189 self.name = name
190 self.acl_class = self.AclClassMap[self.name]
191 self.canned_acls = self.CannedAclsMap[self.name]
192 self._credential_expiry_time = None
193
194 # Load shared credentials file if it exists
195 shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
196 self.shared_credentials = Config(do_load=False)
197 if os.path.isfile(shared_path):
198 self.shared_credentials.load_from_path(shared_path)
199
200 self.get_credentials(access_key, secret_key, security_token, profile_name)
201 self.configure_headers()
202 self.configure_errors()
203
204 # Allow config file to override default host and port.
205 host_opt_name = '%s_host' % self.HostKeyMap[self.name]
206 if config.has_option('Credentials', host_opt_name):
207 self.host = config.get('Credentials', host_opt_name)
208 port_opt_name = '%s_port' % self.HostKeyMap[self.name]
209 if config.has_option('Credentials', port_opt_name):
210 self.port = config.getint('Credentials', port_opt_name)
211 host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
212 if config.has_option('Credentials', host_header_opt_name):
213 self.host_header = config.get('Credentials', host_header_opt_name)
214
215 def get_access_key(self):
216 if self._credentials_need_refresh():
217 self._populate_keys_from_metadata_server()
218 return self._access_key
219
220 def set_access_key(self, value):
221 self._access_key = value
222
223 access_key = property(get_access_key, set_access_key)
224
225 def get_secret_key(self):
226 if self._credentials_need_refresh():
227 self._populate_keys_from_metadata_server()
228 return self._secret_key
229
230 def set_secret_key(self, value):
231 self._secret_key = value
232
233 secret_key = property(get_secret_key, set_secret_key)
234
235 def get_security_token(self):
236 if self._credentials_need_refresh():
237 self._populate_keys_from_metadata_server()
238 return self._security_token
239
240 def set_security_token(self, value):
241 self._security_token = value
242
243 security_token = property(get_security_token, set_security_token)
244
245 def _credentials_need_refresh(self):
246 if self._credential_expiry_time is None:
247 return False
248 else:
249 # The credentials should be refreshed if they're going to expire
250 # in less than 5 minutes.
251 delta = self._credential_expiry_time - datetime.utcnow()
252 # python2.6 does not have timedelta.total_seconds() so we have
253 # to calculate this ourselves. This is straight from the
254 # datetime docs.
255 seconds_left = (
256 (delta.microseconds + (delta.seconds + delta.days * 24 * 3600)
257 * 10 ** 6) / 10 ** 6)
258 if seconds_left < (5 * 60):
259 boto.log.debug("Credentials need to be refreshed.")
260 return True
261 else:
262 return False
263
264 def get_credentials(self, access_key=None, secret_key=None,
265 security_token=None, profile_name=None):
266 access_key_name, secret_key_name, security_token_name, \
267 profile_name_name = self.CredentialMap[self.name]
268
269 # Load profile from shared environment variable if it was not
270 # already passed in and the environment variable exists
271 if profile_name is None and profile_name_name is not None and \
272 profile_name_name.upper() in os.environ:
273 profile_name = os.environ[profile_name_name.upper()]
274
275 shared = self.shared_credentials
276
277 if access_key is not None:
278 self.access_key = access_key
279 boto.log.debug("Using access key provided by client.")
280 elif access_key_name.upper() in os.environ:
281 self.access_key = os.environ[access_key_name.upper()]
282 boto.log.debug("Using access key found in environment variable.")
283 elif profile_name is not None:
284 if shared.has_option(profile_name, access_key_name):
285 self.access_key = shared.get(profile_name, access_key_name)
286 boto.log.debug("Using access key found in shared credential "
287 "file for profile %s." % profile_name)
288 elif config.has_option("profile %s" % profile_name,
289 access_key_name):
290 self.access_key = config.get("profile %s" % profile_name,
291 access_key_name)
292 boto.log.debug("Using access key found in config file: "
293 "profile %s." % profile_name)
294 else:
295 raise ProfileNotFoundError('Profile "%s" not found!' %
296 profile_name)
297 elif shared.has_option('default', access_key_name):
298 self.access_key = shared.get('default', access_key_name)
299 boto.log.debug("Using access key found in shared credential file.")
300 elif config.has_option('Credentials', access_key_name):
301 self.access_key = config.get('Credentials', access_key_name)
302 boto.log.debug("Using access key found in config file.")
303
304 if secret_key is not None:
305 self.secret_key = secret_key
306 boto.log.debug("Using secret key provided by client.")
307 elif secret_key_name.upper() in os.environ:
308 self.secret_key = os.environ[secret_key_name.upper()]
309 boto.log.debug("Using secret key found in environment variable.")
310 elif profile_name is not None:
311 if shared.has_option(profile_name, secret_key_name):
312 self.secret_key = shared.get(profile_name, secret_key_name)
313 boto.log.debug("Using secret key found in shared credential "
314 "file for profile %s." % profile_name)
315 elif config.has_option("profile %s" % profile_name, secret_key_name):
316 self.secret_key = config.get("profile %s" % profile_name,
317 secret_key_name)
318 boto.log.debug("Using secret key found in config file: "
319 "profile %s." % profile_name)
320 else:
321 raise ProfileNotFoundError('Profile "%s" not found!' %
322 profile_name)
323 elif shared.has_option('default', secret_key_name):
324 self.secret_key = shared.get('default', secret_key_name)
325 boto.log.debug("Using secret key found in shared credential file.")
326 elif config.has_option('Credentials', secret_key_name):
327 self.secret_key = config.get('Credentials', secret_key_name)
328 boto.log.debug("Using secret key found in config file.")
329 elif config.has_option('Credentials', 'keyring'):
330 keyring_name = config.get('Credentials', 'keyring')
331 try:
332 import keyring
333 except ImportError:
334 boto.log.error("The keyring module could not be imported. "
335 "For keyring support, install the keyring "
336 "module.")
337 raise
338 self.secret_key = keyring.get_password(
339 keyring_name, self.access_key)
340 boto.log.debug("Using secret key found in keyring.")
341
342 if security_token is not None:
343 self.security_token = security_token
344 boto.log.debug("Using security token provided by client.")
345 elif ((security_token_name is not None) and
346 (access_key is None) and (secret_key is None)):
347 # Only provide a token from the environment/config if the
348 # caller did not specify a key and secret. Otherwise an
349 # environment/config token could be paired with a
350 # different set of credentials provided by the caller
351 if security_token_name.upper() in os.environ:
352 self.security_token = os.environ[security_token_name.upper()]
353 boto.log.debug("Using security token found in environment"
354 " variable.")
355 elif shared.has_option(profile_name or 'default',
356 security_token_name):
357 self.security_token = shared.get(profile_name or 'default',
358 security_token_name)
359 boto.log.debug("Using security token found in shared "
360 "credential file.")
361 elif profile_name is not None:
362 if config.has_option("profile %s" % profile_name,
363 security_token_name):
364 boto.log.debug("config has option")
365 self.security_token = config.get("profile %s" % profile_name,
366 security_token_name)
367 boto.log.debug("Using security token found in config file: "
368 "profile %s." % profile_name)
369 elif config.has_option('Credentials', security_token_name):
370 self.security_token = config.get('Credentials',
371 security_token_name)
372 boto.log.debug("Using security token found in config file.")
373
374 if ((self._access_key is None or self._secret_key is None) and
375 self.MetadataServiceSupport[self.name]):
376 self._populate_keys_from_metadata_server()
377 self._secret_key = self._convert_key_to_str(self._secret_key)
378
379 def _populate_keys_from_metadata_server(self):
380 # get_instance_metadata is imported here because of a circular
381 # dependency.
382 boto.log.debug("Retrieving credentials from metadata server.")
383 from boto.utils import get_instance_metadata
384 timeout = config.getfloat('Boto', 'metadata_service_timeout', 1.0)
385 attempts = config.getint('Boto', 'metadata_service_num_attempts', 1)
386 # The num_retries arg is actually the total number of attempts made,
387 # so the config options is named *_num_attempts to make this more
388 # clear to users.
389 metadata = get_instance_metadata(
390 timeout=timeout, num_retries=attempts,
391 data='meta-data/iam/security-credentials/')
392 if metadata:
393 # I'm assuming there's only one role on the instance profile.
394 security = list(metadata.values())[0]
395 self._access_key = security['AccessKeyId']
396 self._secret_key = self._convert_key_to_str(security['SecretAccessKey'])
397 self._security_token = security['Token']
398 expires_at = security['Expiration']
399 self._credential_expiry_time = datetime.strptime(
400 expires_at, "%Y-%m-%dT%H:%M:%SZ")
401 boto.log.debug("Retrieved credentials will expire in %s at: %s",
402 self._credential_expiry_time - datetime.now(), expires_at)
403
404 def _convert_key_to_str(self, key):
405 if isinstance(key, six.text_type):
406 # the secret key must be bytes and not unicode to work
407 # properly with hmac.new (see http://bugs.python.org/issue5285)
408 return str(key)
409 return key
410
411 def configure_headers(self):
412 header_info_map = self.HeaderInfoMap[self.name]
413 self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY]
414 self.header_prefix = header_info_map[HEADER_PREFIX_KEY]
415 self.acl_header = header_info_map[ACL_HEADER_KEY]
416 self.auth_header = header_info_map[AUTH_HEADER_KEY]
417 self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY]
418 self.copy_source_version_id = header_info_map[
419 COPY_SOURCE_VERSION_ID_HEADER_KEY]
420 self.copy_source_range_header = header_info_map[
421 COPY_SOURCE_RANGE_HEADER_KEY]
422 self.date_header = header_info_map[DATE_HEADER_KEY]
423 self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY]
424 self.metadata_directive_header = (
425 header_info_map[METADATA_DIRECTIVE_HEADER_KEY])
426 self.security_token_header = header_info_map[SECURITY_TOKEN_HEADER_KEY]
427 self.resumable_upload_header = (
428 header_info_map[RESUMABLE_UPLOAD_HEADER_KEY])
429 self.server_side_encryption_header = header_info_map[SERVER_SIDE_ENCRYPTION_KEY]
430 self.storage_class_header = header_info_map[STORAGE_CLASS_HEADER_KEY]
431 self.version_id = header_info_map[VERSION_ID_HEADER_KEY]
432 self.mfa_header = header_info_map[MFA_HEADER_KEY]
433 self.restore_header = header_info_map[RESTORE_HEADER_KEY]
434
435 def configure_errors(self):
436 error_map = self.ErrorMap[self.name]
437 self.storage_copy_error = error_map[STORAGE_COPY_ERROR]
438 self.storage_create_error = error_map[STORAGE_CREATE_ERROR]
439 self.storage_data_error = error_map[STORAGE_DATA_ERROR]
440 self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR]
441 self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR]
442
443 def get_provider_name(self):
444 return self.HostKeyMap[self.name]
445
446 def supports_chunked_transfer(self):
447 return self.ChunkedTransferSupport[self.name]
448
449
450 # Static utility method for getting default Provider.
451 def get_default():
452 return Provider('aws')