1 """
2 connection operations
3
4 Connection instances are used to communicate with the remote service at
5 the account level creating, listing and deleting Containers, and returning
6 Container instances.
7
8 See COPYING for license information.
9 """
10
11 import socket
12 import os
13 from urllib import quote
14 from httplib import HTTPSConnection, HTTPConnection, HTTPException
15 from container import Container, ContainerResults
16 from utils import parse_url
17 from errors import ResponseError, NoSuchContainer, ContainerNotEmpty, \
18 InvalidContainerName, CDNNotEnabled
19 from Queue import Queue, Empty, Full
20 from time import time
21 import consts
22 from authentication import Authentication
23 from fjson import json_loads
24
25
26
27
28
29
31 """
32 Manages the connection to the storage system and serves as a factory
33 for Container instances.
34
35 @undocumented: cdn_connect
36 @undocumented: http_connect
37 @undocumented: cdn_request
38 @undocumented: make_request
39 @undocumented: _check_container_name
40 """
41
42 - def __init__(self, username=None, api_key=None, **kwargs):
43 """
44 Accepts keyword arguments for Mosso username and api key.
45 Optionally, you can omit these keywords and supply an
46 Authentication object using the auth keyword. Setting the argument
47 servicenet to True will make use of Rackspace servicenet network.
48
49 @type username: str
50 @param username: a Mosso username
51 @type api_key: str
52 @param api_key: a Mosso API key
53 @type servicenet: bool
54 @param servicenet: Use Rackspace servicenet to access Cloud Files.
55 @type cdn_log_retention: bool
56 @param cdn_log_retention: set logs retention for this cdn enabled
57 container.
58 """
59 self.cdn_enabled = False
60 self.cdn_args = None
61 self.connection_args = None
62 self.cdn_connection = None
63 self.connection = None
64 self.token = None
65 self.debuglevel = int(kwargs.get('debuglevel', 0))
66 self.servicenet = kwargs.get('servicenet', False)
67
68
69
70 if not 'servicenet' in kwargs \
71 and 'RACKSPACE_SERVICENET' in os.environ:
72 self.servicenet = True
73
74 socket.setdefaulttimeout = int(kwargs.get('timeout', 5))
75 self.auth = 'auth' in kwargs and kwargs['auth'] or None
76
77 if not self.auth:
78 authurl = kwargs.get('authurl', consts.default_authurl)
79 if username and api_key and authurl:
80 self.auth = Authentication(username, api_key, authurl)
81 else:
82 raise TypeError("Incorrect or invalid arguments supplied")
83
84 self._authenticate()
85
87 """
88 Authenticate and setup this instance with the values returned.
89 """
90 (url, self.cdn_url, self.token) = self.auth.authenticate()
91 url = self._set_storage_url(url)
92 self.connection_args = parse_url(url)
93 self.conn_class = self.connection_args[3] and HTTPSConnection or \
94 HTTPConnection
95 self.http_connect()
96 if self.cdn_url:
97 self.cdn_connect()
98
100 if self.servicenet:
101 return "https://snet-%s" % url.replace("https://", "")
102 return url
103
105 """
106 Setup the http connection instance for the CDN service.
107 """
108 (host, port, cdn_uri, is_ssl) = parse_url(self.cdn_url)
109 conn_class = is_ssl and HTTPSConnection or HTTPConnection
110 self.cdn_connection = conn_class(host, port)
111 self.cdn_enabled = True
112
114 """
115 Setup the http connection instance.
116 """
117 (host, port, self.uri, is_ssl) = self.connection_args
118 self.connection = self.conn_class(host, port=port)
119 self.connection.set_debuglevel(self.debuglevel)
120
121 - def cdn_request(self, method, path=[], data='', hdrs=None):
122 """
123 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
124 metadata dicts, performs an http request against the CDN service.
125 """
126 if not self.cdn_enabled:
127 raise CDNNotEnabled()
128
129 path = '/%s/%s' % \
130 (self.uri.rstrip('/'), '/'.join([quote(i) for i in path]))
131 headers = {'Content-Length': len(data),
132 'User-Agent': consts.user_agent,
133 'X-Auth-Token': self.token}
134 if isinstance(hdrs, dict):
135 headers.update(hdrs)
136
137
138 self.cdn_connection.request(method, path, data, headers)
139
140 def retry_request():
141 '''Re-connect and re-try a failed request once'''
142 self.cdn_connect()
143 self.cdn_connection.request(method, path, data, headers)
144 return self.cdn_connection.getresponse()
145
146 try:
147 response = self.cdn_connection.getresponse()
148 except HTTPException:
149 response = retry_request()
150
151 if response.status == 401:
152 self._authenticate()
153 response = retry_request()
154
155 return response
156
157 - def make_request(self, method, path=[], data='', hdrs=None, parms=None):
158 """
159 Given a method (i.e. GET, PUT, POST, etc), a path, data, header and
160 metadata dicts, and an optional dictionary of query parameters,
161 performs an http request.
162 """
163 path = '/%s/%s' % \
164 (self.uri.rstrip('/'), '/'.join([quote(i) for i in path]))
165
166 if isinstance(parms, dict) and parms:
167 query_args = \
168 ['%s=%s' % (quote(x),
169 quote(str(y))) for (x, y) in parms.items()]
170 path = '%s?%s' % (path, '&'.join(query_args))
171
172 headers = {'Content-Length': len(data),
173 'User-Agent': consts.user_agent,
174 'X-Auth-Token': self.token}
175 isinstance(hdrs, dict) and headers.update(hdrs)
176
177 def retry_request():
178 '''Re-connect and re-try a failed request once'''
179 self.http_connect()
180 self.connection.request(method, path, data, headers)
181 return self.connection.getresponse()
182
183 try:
184 self.connection.request(method, path, data, headers)
185 response = self.connection.getresponse()
186 except HTTPException:
187 response = retry_request()
188
189 if response.status == 401:
190 self._authenticate()
191 response = retry_request()
192
193 return response
194
196 """
197 Return tuple for number of containers and total bytes in the account
198
199 >>> connection.get_info()
200 (5, 2309749)
201
202 @rtype: tuple
203 @return: a tuple containing the number of containers and total bytes
204 used by the account
205 """
206 response = self.make_request('HEAD')
207 count = size = None
208 for hdr in response.getheaders():
209 if hdr[0].lower() == 'x-account-container-count':
210 try:
211 count = int(hdr[1])
212 except ValueError:
213 count = 0
214 if hdr[0].lower() == 'x-account-bytes-used':
215 try:
216 size = int(hdr[1])
217 except ValueError:
218 size = 0
219 buff = response.read()
220 if (response.status < 200) or (response.status > 299):
221 raise ResponseError(response.status, response.reason)
222 return (count, size)
223
225 if not container_name or \
226 '/' in container_name or \
227 len(container_name) > consts.container_name_limit:
228 raise InvalidContainerName(container_name)
229
231 """
232 Given a container name, returns a L{Container} item, creating a new
233 Container if one does not already exist.
234
235 >>> connection.create_container('new_container')
236 <cloudfiles.container.Container object at 0xb77d628c>
237
238 @param container_name: name of the container to create
239 @type container_name: str
240 @rtype: L{Container}
241 @return: an object representing the newly created container
242 """
243 self._check_container_name(container_name)
244
245 response = self.make_request('PUT', [container_name])
246 buff = response.read()
247 if (response.status < 200) or (response.status > 299):
248 raise ResponseError(response.status, response.reason)
249 return Container(self, container_name)
250
252 """
253 Given a container name, delete it.
254
255 >>> connection.delete_container('old_container')
256
257 @param container_name: name of the container to delete
258 @type container_name: str
259 """
260 if isinstance(container_name, Container):
261 container_name = container_name.name
262 self._check_container_name(container_name)
263
264 response = self.make_request('DELETE', [container_name])
265 buff = response.read()
266
267 if (response.status == 409):
268 raise ContainerNotEmpty(container_name)
269 elif (response.status < 200) or (response.status > 299):
270 raise ResponseError(response.status, response.reason)
271
272 if self.cdn_enabled:
273 response = self.cdn_request('POST', [container_name],
274 hdrs={'X-CDN-Enabled': 'False'})
275
277 """
278 Returns a Container item result set.
279
280 >>> connection.get_all_containers()
281 ContainerResults: 4 containers
282 >>> print ', '.join([container.name for container in
283 connection.get_all_containers()])
284 new_container, old_container, pictures, music
285
286 @rtype: L{ContainerResults}
287 @return: an iterable set of objects representing all containers on the
288 account
289 @param limit: number of results to return, up to 10,000
290 @type limit: int
291 @param marker: return only results whose name is greater than "marker"
292 @type marker: str
293 """
294 if limit:
295 parms['limit'] = limit
296 if marker:
297 parms['marker'] = marker
298 return ContainerResults(self, self.list_containers_info(**parms))
299
301 """
302 Return a single Container item for the given Container.
303
304 >>> connection.get_container('old_container')
305 <cloudfiles.container.Container object at 0xb77d628c>
306 >>> container = connection.get_container('old_container')
307 >>> container.size_used
308 23074
309
310 @param container_name: name of the container to create
311 @type container_name: str
312 @rtype: L{Container}
313 @return: an object representing the container
314 """
315 self._check_container_name(container_name)
316
317 response = self.make_request('HEAD', [container_name])
318 count = size = None
319 for hdr in response.getheaders():
320 if hdr[0].lower() == 'x-container-object-count':
321 try:
322 count = int(hdr[1])
323 except ValueError:
324 count = 0
325 if hdr[0].lower() == 'x-container-bytes-used':
326 try:
327 size = int(hdr[1])
328 except ValueError:
329 size = 0
330 buff = response.read()
331 if response.status == 404:
332 raise NoSuchContainer(container_name)
333 if (response.status < 200) or (response.status > 299):
334 raise ResponseError(response.status, response.reason)
335 return Container(self, container_name, count, size)
336
338 """
339 Returns a list of containers that have been published to the CDN.
340
341 >>> connection.list_public_containers()
342 ['container1', 'container2', 'container3']
343
344 @rtype: list(str)
345 @return: a list of all CDN-enabled container names as strings
346 """
347 response = self.cdn_request('GET', [''])
348 if (response.status < 200) or (response.status > 299):
349 buff = response.read()
350 raise ResponseError(response.status, response.reason)
351 return response.read().splitlines()
352
354 """
355 Returns a list of Containers, including object count and size.
356
357 >>> connection.list_containers_info()
358 [{u'count': 510, u'bytes': 2081717, u'name': u'new_container'},
359 {u'count': 12, u'bytes': 23074, u'name': u'old_container'},
360 {u'count': 0, u'bytes': 0, u'name': u'container1'},
361 {u'count': 0, u'bytes': 0, u'name': u'container2'},
362 {u'count': 0, u'bytes': 0, u'name': u'container3'},
363 {u'count': 3, u'bytes': 2306, u'name': u'test'}]
364
365 @rtype: list({"name":"...", "count":..., "bytes":...})
366 @return: a list of all container info as dictionaries with the
367 keys "name", "count", and "bytes"
368 @param limit: number of results to return, up to 10,000
369 @type limit: int
370 @param marker: return only results whose name is greater than "marker"
371 @type marker: str
372 """
373 if limit:
374 parms['limit'] = limit
375 if marker:
376 parms['marker'] = marker
377 parms['format'] = 'json'
378 response = self.make_request('GET', [''], parms=parms)
379 if (response.status < 200) or (response.status > 299):
380 buff = response.read()
381 raise ResponseError(response.status, response.reason)
382 return json_loads(response.read())
383
385 """
386 Returns a list of Containers.
387
388 >>> connection.list_containers()
389 ['new_container',
390 'old_container',
391 'container1',
392 'container2',
393 'container3',
394 'test']
395
396 @rtype: list(str)
397 @return: a list of all containers names as strings
398 @param limit: number of results to return, up to 10,000
399 @type limit: int
400 @param marker: return only results whose name is greater than "marker"
401 @type marker: str
402 """
403 if limit:
404 parms['limit'] = limit
405 if marker:
406 parms['marker'] = marker
407 response = self.make_request('GET', [''], parms=parms)
408 if (response.status < 200) or (response.status > 299):
409 buff = response.read()
410 raise ResponseError(response.status, response.reason)
411 return response.read().splitlines()
412
414 """
415 Container objects can be grabbed from a connection using index
416 syntax.
417
418 >>> container = conn['old_container']
419 >>> container.size_used
420 23074
421
422 @rtype: L{Container}
423 @return: an object representing the container
424 """
425 return self.get_container(key)
426
427
429 """
430 A thread-safe connection pool object.
431
432 This component isn't required when using the cloudfiles library, but it may
433 be useful when building threaded applications.
434 """
435
436 - def __init__(self, username=None, api_key=None, **kwargs):
437 auth = kwargs.get('auth', None)
438 self.timeout = kwargs.get('timeout', 5)
439 self.connargs = {'username': username, 'api_key': api_key}
440 poolsize = kwargs.get('poolsize', 10)
441 Queue.__init__(self, poolsize)
442
444 """
445 Return a cloudfiles connection object.
446
447 @rtype: L{Connection}
448 @return: a cloudfiles connection object
449 """
450 try:
451 (create, connobj) = Queue.get(self, block=0)
452 except Empty:
453 connobj = Connection(**self.connargs)
454 return connobj
455
456 - def put(self, connobj):
457 """
458 Place a cloudfiles connection object back into the pool.
459
460 @param connobj: a cloudfiles connection object
461 @type connobj: L{Connection}
462 """
463 try:
464 Queue.put(self, (time(), connobj), block=0)
465 except Full:
466 del connobj
467
468
469