Package cloudfiles :: Module connection
[frames] | no frames]

Source Code for Module cloudfiles.connection

  1  """ 
  2  connection operations 
  3   
  4  Connection instances are used to communicate with the remote service at 
  5  the account level creating, listing and deleting Containers, and returning 
  6  Container instances. 
  7   
  8  See COPYING for license information. 
  9  """ 
 10   
 11  import  socket 
 12  import  os 
 13  from    urllib    import quote 
 14  from    httplib   import HTTPSConnection, HTTPConnection, HTTPException 
 15  from    container import Container, ContainerResults 
 16  from    utils     import parse_url 
 17  from    errors    import ResponseError, NoSuchContainer, ContainerNotEmpty, \ 
 18                           InvalidContainerName, CDNNotEnabled 
 19  from    Queue     import Queue, Empty, Full 
 20  from    time      import time 
 21  import  consts 
 22  from    authentication import Authentication 
 23  from    fjson     import json_loads 
 24   
 25  # Because HTTPResponse objects *have* to have read() called on them 
 26  # before they can be used again ... 
 27  # pylint: disable-msg=W0612 
 28   
 29   
30 -class Connection(object):
31 """ 32 Manages the connection to the storage system and serves as a factory 33 for Container instances. 34 35 @undocumented: cdn_connect 36 @undocumented: http_connect 37 @undocumented: cdn_request 38 @undocumented: make_request 39 @undocumented: _check_container_name 40 """ 41
42 - def __init__(self, username=None, api_key=None, **kwargs):
43 """ 44 Accepts keyword arguments for Mosso username and api key. 45 Optionally, you can omit these keywords and supply an 46 Authentication object using the auth keyword. Setting the argument 47 servicenet to True will make use of Rackspace servicenet network. 48 49 @type username: str 50 @param username: a Mosso username 51 @type api_key: str 52 @param api_key: a Mosso API key 53 @type servicenet: bool 54 @param servicenet: Use Rackspace servicenet to access Cloud Files. 55 @type cdn_log_retention: bool 56 @param cdn_log_retention: set logs retention for this cdn enabled 57 container. 58 """ 59 self.cdn_enabled = False 60 self.cdn_args = None 61 self.connection_args = None 62 self.cdn_connection = None 63 self.connection = None 64 self.token = None 65 self.debuglevel = int(kwargs.get('debuglevel', 0)) 66 self.servicenet = kwargs.get('servicenet', False) 67 68 # if the environement variable RACKSPACE_SERVICENET is set (to 69 # anything) it will automatically set servicenet=True 70 if not 'servicenet' in kwargs \ 71 and 'RACKSPACE_SERVICENET' in os.environ: 72 self.servicenet = True 73 74 socket.setdefaulttimeout = int(kwargs.get('timeout', 5)) 75 self.auth = 'auth' in kwargs and kwargs['auth'] or None 76 77 if not self.auth: 78 authurl = kwargs.get('authurl', consts.default_authurl) 79 if username and api_key and authurl: 80 self.auth = Authentication(username, api_key, authurl) 81 else: 82 raise TypeError("Incorrect or invalid arguments supplied") 83 84 self._authenticate()
85
86 - def _authenticate(self):
87 """ 88 Authenticate and setup this instance with the values returned. 89 """ 90 (url, self.cdn_url, self.token) = self.auth.authenticate() 91 url = self._set_storage_url(url) 92 self.connection_args = parse_url(url) 93 self.conn_class = self.connection_args[3] and HTTPSConnection or \ 94 HTTPConnection 95 self.http_connect() 96 if self.cdn_url: 97 self.cdn_connect()
98
99 - def _set_storage_url(self, url):
100 if self.servicenet: 101 return "https://snet-%s" % url.replace("https://", "") 102 return url
103
104 - def cdn_connect(self):
105 """ 106 Setup the http connection instance for the CDN service. 107 """ 108 (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url) 109 conn_class = is_ssl and HTTPSConnection or HTTPConnection 110 self.cdn_connection = conn_class(host, port) 111 self.cdn_enabled = True
112
113 - def http_connect(self):
114 """ 115 Setup the http connection instance. 116 """ 117 (host, port, self.uri, is_ssl) = self.connection_args 118 self.connection = self.conn_class(host, port=port) 119 self.connection.set_debuglevel(self.debuglevel)
120
121 - def cdn_request(self, method, path=[], data='', hdrs=None):
122 """ 123 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and 124 metadata dicts, performs an http request against the CDN service. 125 """ 126 if not self.cdn_enabled: 127 raise CDNNotEnabled() 128 129 path = '/%s/%s' % \ 130 (self.uri.rstrip('/'), '/'.join([quote(i) for i in path])) 131 headers = {'Content-Length': len(data), 132 'User-Agent': consts.user_agent, 133 'X-Auth-Token': self.token} 134 if isinstance(hdrs, dict): 135 headers.update(hdrs) 136 137 # Send the request 138 self.cdn_connection.request(method, path, data, headers) 139 140 def retry_request(): 141 '''Re-connect and re-try a failed request once''' 142 self.cdn_connect() 143 self.cdn_connection.request(method, path, data, headers) 144 return self.cdn_connection.getresponse()
145 146 try: 147 response = self.cdn_connection.getresponse() 148 except HTTPException: 149 response = retry_request() 150 151 if response.status == 401: 152 self._authenticate() 153 response = retry_request() 154 155 return response
156
157 - def make_request(self, method, path=[], data='', hdrs=None, parms=None):
158 """ 159 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and 160 metadata dicts, and an optional dictionary of query parameters, 161 performs an http request. 162 """ 163 path = '/%s/%s' % \ 164 (self.uri.rstrip('/'), '/'.join([quote(i) for i in path])) 165 166 if isinstance(parms, dict) and parms: 167 query_args = \ 168 ['%s=%s' % (quote(x), 169 quote(str(y))) for (x, y) in parms.items()] 170 path = '%s?%s' % (path, '&'.join(query_args)) 171 172 headers = {'Content-Length': len(data), 173 'User-Agent': consts.user_agent, 174 'X-Auth-Token': self.token} 175 isinstance(hdrs, dict) and headers.update(hdrs) 176 177 def retry_request(): 178 '''Re-connect and re-try a failed request once''' 179 self.http_connect() 180 self.connection.request(method, path, data, headers) 181 return self.connection.getresponse()
182 183 try: 184 self.connection.request(method, path, data, headers) 185 response = self.connection.getresponse() 186 except HTTPException: 187 response = retry_request() 188 189 if response.status == 401: 190 self._authenticate() 191 response = retry_request() 192 193 return response 194
195 - def get_info(self):
196 """ 197 Return tuple for number of containers and total bytes in the account 198 199 >>> connection.get_info() 200 (5, 2309749) 201 202 @rtype: tuple 203 @return: a tuple containing the number of containers and total bytes 204 used by the account 205 """ 206 response = self.make_request('HEAD') 207 count = size = None 208 for hdr in response.getheaders(): 209 if hdr[0].lower() == 'x-account-container-count': 210 try: 211 count = int(hdr[1]) 212 except ValueError: 213 count = 0 214 if hdr[0].lower() == 'x-account-bytes-used': 215 try: 216 size = int(hdr[1]) 217 except ValueError: 218 size = 0 219 buff = response.read() 220 if (response.status < 200) or (response.status > 299): 221 raise ResponseError(response.status, response.reason) 222 return (count, size)
223
224 - def _check_container_name(self, container_name):
225 if not container_name or \ 226 '/' in container_name or \ 227 len(container_name) > consts.container_name_limit: 228 raise InvalidContainerName(container_name)
229
230 - def create_container(self, container_name):
231 """ 232 Given a container name, returns a L{Container} item, creating a new 233 Container if one does not already exist. 234 235 >>> connection.create_container('new_container') 236 <cloudfiles.container.Container object at 0xb77d628c> 237 238 @param container_name: name of the container to create 239 @type container_name: str 240 @rtype: L{Container} 241 @return: an object representing the newly created container 242 """ 243 self._check_container_name(container_name) 244 245 response = self.make_request('PUT', [container_name]) 246 buff = response.read() 247 if (response.status < 200) or (response.status > 299): 248 raise ResponseError(response.status, response.reason) 249 return Container(self, container_name)
250
251 - def delete_container(self, container_name):
252 """ 253 Given a container name, delete it. 254 255 >>> connection.delete_container('old_container') 256 257 @param container_name: name of the container to delete 258 @type container_name: str 259 """ 260 if isinstance(container_name, Container): 261 container_name = container_name.name 262 self._check_container_name(container_name) 263 264 response = self.make_request('DELETE', [container_name]) 265 buff = response.read() 266 267 if (response.status == 409): 268 raise ContainerNotEmpty(container_name) 269 elif (response.status < 200) or (response.status > 299): 270 raise ResponseError(response.status, response.reason) 271 272 if self.cdn_enabled: 273 response = self.cdn_request('POST', [container_name], 274 hdrs={'X-CDN-Enabled': 'False'})
275
276 - def get_all_containers(self, limit=None, marker=None, **parms):
277 """ 278 Returns a Container item result set. 279 280 >>> connection.get_all_containers() 281 ContainerResults: 4 containers 282 >>> print ', '.join([container.name for container in 283 connection.get_all_containers()]) 284 new_container, old_container, pictures, music 285 286 @rtype: L{ContainerResults} 287 @return: an iterable set of objects representing all containers on the 288 account 289 @param limit: number of results to return, up to 10,000 290 @type limit: int 291 @param marker: return only results whose name is greater than "marker" 292 @type marker: str 293 """ 294 if limit: 295 parms['limit'] = limit 296 if marker: 297 parms['marker'] = marker 298 return ContainerResults(self, self.list_containers_info(**parms))
299
300 - def get_container(self, container_name):
301 """ 302 Return a single Container item for the given Container. 303 304 >>> connection.get_container('old_container') 305 <cloudfiles.container.Container object at 0xb77d628c> 306 >>> container = connection.get_container('old_container') 307 >>> container.size_used 308 23074 309 310 @param container_name: name of the container to create 311 @type container_name: str 312 @rtype: L{Container} 313 @return: an object representing the container 314 """ 315 self._check_container_name(container_name) 316 317 response = self.make_request('HEAD', [container_name]) 318 count = size = None 319 for hdr in response.getheaders(): 320 if hdr[0].lower() == 'x-container-object-count': 321 try: 322 count = int(hdr[1]) 323 except ValueError: 324 count = 0 325 if hdr[0].lower() == 'x-container-bytes-used': 326 try: 327 size = int(hdr[1]) 328 except ValueError: 329 size = 0 330 buff = response.read() 331 if response.status == 404: 332 raise NoSuchContainer(container_name) 333 if (response.status < 200) or (response.status > 299): 334 raise ResponseError(response.status, response.reason) 335 return Container(self, container_name, count, size)
336
337 - def list_public_containers(self):
338 """ 339 Returns a list of containers that have been published to the CDN. 340 341 >>> connection.list_public_containers() 342 ['container1', 'container2', 'container3'] 343 344 @rtype: list(str) 345 @return: a list of all CDN-enabled container names as strings 346 """ 347 response = self.cdn_request('GET', ['']) 348 if (response.status < 200) or (response.status > 299): 349 buff = response.read() 350 raise ResponseError(response.status, response.reason) 351 return response.read().splitlines()
352
353 - def list_containers_info(self, limit=None, marker=None, **parms):
354 """ 355 Returns a list of Containers, including object count and size. 356 357 >>> connection.list_containers_info() 358 [{u'count': 510, u'bytes': 2081717, u'name': u'new_container'}, 359 {u'count': 12, u'bytes': 23074, u'name': u'old_container'}, 360 {u'count': 0, u'bytes': 0, u'name': u'container1'}, 361 {u'count': 0, u'bytes': 0, u'name': u'container2'}, 362 {u'count': 0, u'bytes': 0, u'name': u'container3'}, 363 {u'count': 3, u'bytes': 2306, u'name': u'test'}] 364 365 @rtype: list({"name":"...", "count":..., "bytes":...}) 366 @return: a list of all container info as dictionaries with the 367 keys "name", "count", and "bytes" 368 @param limit: number of results to return, up to 10,000 369 @type limit: int 370 @param marker: return only results whose name is greater than "marker" 371 @type marker: str 372 """ 373 if limit: 374 parms['limit'] = limit 375 if marker: 376 parms['marker'] = marker 377 parms['format'] = 'json' 378 response = self.make_request('GET', [''], parms=parms) 379 if (response.status < 200) or (response.status > 299): 380 buff = response.read() 381 raise ResponseError(response.status, response.reason) 382 return json_loads(response.read())
383
384 - def list_containers(self, limit=None, marker=None, **parms):
385 """ 386 Returns a list of Containers. 387 388 >>> connection.list_containers() 389 ['new_container', 390 'old_container', 391 'container1', 392 'container2', 393 'container3', 394 'test'] 395 396 @rtype: list(str) 397 @return: a list of all containers names as strings 398 @param limit: number of results to return, up to 10,000 399 @type limit: int 400 @param marker: return only results whose name is greater than "marker" 401 @type marker: str 402 """ 403 if limit: 404 parms['limit'] = limit 405 if marker: 406 parms['marker'] = marker 407 response = self.make_request('GET', [''], parms=parms) 408 if (response.status < 200) or (response.status > 299): 409 buff = response.read() 410 raise ResponseError(response.status, response.reason) 411 return response.read().splitlines()
412
413 - def __getitem__(self, key):
414 """ 415 Container objects can be grabbed from a connection using index 416 syntax. 417 418 >>> container = conn['old_container'] 419 >>> container.size_used 420 23074 421 422 @rtype: L{Container} 423 @return: an object representing the container 424 """ 425 return self.get_container(key)
426 427
428 -class ConnectionPool(Queue):
429 """ 430 A thread-safe connection pool object. 431 432 This component isn't required when using the cloudfiles library, but it may 433 be useful when building threaded applications. 434 """ 435
436 - def __init__(self, username=None, api_key=None, **kwargs):
437 auth = kwargs.get('auth', None) 438 self.timeout = kwargs.get('timeout', 5) 439 self.connargs = {'username': username, 'api_key': api_key} 440 poolsize = kwargs.get('poolsize', 10) 441 Queue.__init__(self, poolsize)
442
443 - def get(self):
444 """ 445 Return a cloudfiles connection object. 446 447 @rtype: L{Connection} 448 @return: a cloudfiles connection object 449 """ 450 try: 451 (create, connobj) = Queue.get(self, block=0) 452 except Empty: 453 connobj = Connection(**self.connargs) 454 return connobj
455
456 - def put(self, connobj):
457 """ 458 Place a cloudfiles connection object back into the pool. 459 460 @param connobj: a cloudfiles connection object 461 @type connobj: L{Connection} 462 """ 463 try: 464 Queue.put(self, (time(), connobj), block=0) 465 except Full: 466 del connobj
467 468 # vim:set ai sw=4 ts=4 tw=0 expandtab: 469