# Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ # Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # from boto.dynamodb.batch import BatchList from boto.dynamodb.schema import Schema from boto.dynamodb.item import Item from boto.dynamodb import exceptions as dynamodb_exceptions import time class TableBatchGenerator(object): """ A low-level generator used to page through results from batch_get_item operations. :ivar consumed_units: An integer that holds the number of ConsumedCapacityUnits accumulated thus far for this generator. """ def __init__(self, table, keys, attributes_to_get=None, consistent_read=False): self.table = table self.keys = keys self.consumed_units = 0 self.attributes_to_get = attributes_to_get self.consistent_read = consistent_read def _queue_unprocessed(self, res): if u'UnprocessedKeys' not in res: return if self.table.name not in res[u'UnprocessedKeys']: return keys = res[u'UnprocessedKeys'][self.table.name][u'Keys'] for key in keys: h = key[u'HashKeyElement'] r = key[u'RangeKeyElement'] if u'RangeKeyElement' in key else None self.keys.append((h, r)) def __iter__(self): while self.keys: # Build the next batch batch = BatchList(self.table.layer2) batch.add_batch(self.table, self.keys[:100], self.attributes_to_get) res = batch.submit() # parse the results if self.table.name not in res[u'Responses']: continue self.consumed_units += res[u'Responses'][self.table.name][u'ConsumedCapacityUnits'] for elem in res[u'Responses'][self.table.name][u'Items']: yield elem # re-queue un processed keys self.keys = self.keys[100:] self._queue_unprocessed(res) class Table(object): """ An Amazon DynamoDB table. :ivar name: The name of the table. :ivar create_time: The date and time that the table was created. :ivar status: The current status of the table. One of: 'ACTIVE', 'UPDATING', 'DELETING'. :ivar schema: A :class:`boto.dynamodb.schema.Schema` object representing the schema defined for the table. :ivar item_count: The number of items in the table. This value is set only when the Table object is created or refreshed and may not reflect the actual count. :ivar size_bytes: Total size of the specified table, in bytes. Amazon DynamoDB updates this value approximately every six hours. Recent changes might not be reflected in this value. :ivar read_units: The ReadCapacityUnits of the tables Provisioned Throughput. :ivar write_units: The WriteCapacityUnits of the tables Provisioned Throughput. :ivar schema: The Schema object associated with the table. """ def __init__(self, layer2, response): """ :type layer2: :class:`boto.dynamodb.layer2.Layer2` :param layer2: A `Layer2` api object. :type response: dict :param response: The output of `boto.dynamodb.layer1.Layer1.describe_table`. """ self.layer2 = layer2 self._dict = {} self.update_from_response(response) @classmethod def create_from_schema(cls, layer2, name, schema): """Create a Table object. If you know the name and schema of your table, you can create a ``Table`` object without having to make any API calls (normally an API call is made to retrieve the schema of a table). Example usage:: table = Table.create_from_schema( boto.connect_dynamodb(), 'tablename', Schema.create(hash_key=('keyname', 'N'))) :type layer2: :class:`boto.dynamodb.layer2.Layer2` :param layer2: A ``Layer2`` api object. :type name: str :param name: The name of the table. :type schema: :class:`boto.dynamodb.schema.Schema` :param schema: The schema associated with the table. :rtype: :class:`boto.dynamodb.table.Table` :return: A Table object representing the table. """ table = cls(layer2, {'Table': {'TableName': name}}) table._schema = schema return table def __repr__(self): return 'Table(%s)' % self.name @property def name(self): return self._dict['TableName'] @property def create_time(self): return self._dict.get('CreationDateTime', None) @property def status(self): return self._dict.get('TableStatus', None) @property def item_count(self): return self._dict.get('ItemCount', 0) @property def size_bytes(self): return self._dict.get('TableSizeBytes', 0) @property def schema(self): return self._schema @property def read_units(self): try: return self._dict['ProvisionedThroughput']['ReadCapacityUnits'] except KeyError: return None @property def write_units(self): try: return self._dict['ProvisionedThroughput']['WriteCapacityUnits'] except KeyError: return None def update_from_response(self, response): """ Update the state of the Table object based on the response data received from Amazon DynamoDB. """ # 'Table' is from a describe_table call. if 'Table' in response: self._dict.update(response['Table']) # 'TableDescription' is from a create_table call. elif 'TableDescription' in response: self._dict.update(response['TableDescription']) if 'KeySchema' in self._dict: self._schema = Schema(self._dict['KeySchema']) def refresh(self, wait_for_active=False, retry_seconds=5): """ Refresh all of the fields of the Table object by calling the underlying DescribeTable request. :type wait_for_active: bool :param wait_for_active: If True, this command will not return until the table status, as returned from Amazon DynamoDB, is 'ACTIVE'. :type retry_seconds: int :param retry_seconds: If wait_for_active is True, this parameter controls the number of seconds of delay between calls to update_table in Amazon DynamoDB. Default is 5 seconds. """ done = False while not done: response = self.layer2.describe_table(self.name) self.update_from_response(response) if wait_for_active: if self.status == 'ACTIVE': done = True else: time.sleep(retry_seconds) else: done = True def update_throughput(self, read_units, write_units): """ Update the ProvisionedThroughput for the Amazon DynamoDB Table. :type read_units: int :param read_units: The new value for ReadCapacityUnits. :type write_units: int :param write_units: The new value for WriteCapacityUnits. """ self.layer2.update_throughput(self, read_units, write_units) def delete(self): """ Delete this table and all items in it. After calling this the Table objects status attribute will be set to 'DELETING'. """ self.layer2.delete_table(self) def get_item(self, hash_key, range_key=None, attributes_to_get=None, consistent_read=False, item_class=Item): """ Retrieve an existing item from the table. :type hash_key: int|long|float|str|unicode|Binary :param hash_key: The HashKey of the requested item. The type of the value must match the type defined in the schema for the table. :type range_key: int|long|float|str|unicode|Binary :param range_key: The optional RangeKey of the requested item. The type of the value must match the type defined in the schema for the table. :type attributes_to_get: list :param attributes_to_get: A list of attribute names. If supplied, only the specified attribute names will be returned. Otherwise, all attributes will be returned. :type consistent_read: bool :param consistent_read: If True, a consistent read request is issued. Otherwise, an eventually consistent request is issued. :type item_class: Class :param item_class: Allows you to override the class used to generate the items. This should be a subclass of :class:`boto.dynamodb.item.Item` """ return self.layer2.get_item(self, hash_key, range_key, attributes_to_get, consistent_read, item_class) lookup = get_item def has_item(self, hash_key, range_key=None, consistent_read=False): """ Checks the table to see if the Item with the specified ``hash_key`` exists. This may save a tiny bit of time/bandwidth over a straight :py:meth:`get_item` if you have no intention to touch the data that is returned, since this method specifically tells Amazon not to return anything but the Item's key. :type hash_key: int|long|float|str|unicode|Binary :param hash_key: The HashKey of the requested item. The type of the value must match the type defined in the schema for the table. :type range_key: int|long|float|str|unicode|Binary :param range_key: The optional RangeKey of the requested item. The type of the value must match the type defined in the schema for the table. :type consistent_read: bool :param consistent_read: If True, a consistent read request is issued. Otherwise, an eventually consistent request is issued. :rtype: bool :returns: ``True`` if the Item exists, ``False`` if not. """ try: # Attempt to get the key. If it can't be found, it'll raise # an exception. self.get_item(hash_key, range_key=range_key, # This minimizes the size of the response body. attributes_to_get=[hash_key], consistent_read=consistent_read) except dynamodb_exceptions.DynamoDBKeyNotFoundError: # Key doesn't exist. return False return True def new_item(self, hash_key=None, range_key=None, attrs=None, item_class=Item): """ Return an new, unsaved Item which can later be PUT to Amazon DynamoDB. This method has explicit (but optional) parameters for the hash_key and range_key values of the item. You can use these explicit parameters when calling the method, such as:: >>> my_item = my_table.new_item(hash_key='a', range_key=1, attrs={'key1': 'val1', 'key2': 'val2'}) >>> my_item {u'bar': 1, u'foo': 'a', 'key1': 'val1', 'key2': 'val2'} Or, if you prefer, you can simply put the hash_key and range_key in the attrs dictionary itself, like this:: >>> attrs = {'foo': 'a', 'bar': 1, 'key1': 'val1', 'key2': 'val2'} >>> my_item = my_table.new_item(attrs=attrs) >>> my_item {u'bar': 1, u'foo': 'a', 'key1': 'val1', 'key2': 'val2'} The effect is the same. .. note: The explicit parameters take priority over the values in the attrs dict. So, if you have a hash_key or range_key in the attrs dict and you also supply either or both using the explicit parameters, the values in the attrs will be ignored. :type hash_key: int|long|float|str|unicode|Binary :param hash_key: The HashKey of the new item. The type of the value must match the type defined in the schema for the table. :type range_key: int|long|float|str|unicode|Binary :param range_key: The optional RangeKey of the new item. The type of the value must match the type defined in the schema for the table. :type attrs: dict :param attrs: A dictionary of key value pairs used to populate the new item. :type item_class: Class :param item_class: Allows you to override the class used to generate the items. This should be a subclass of :class:`boto.dynamodb.item.Item` """ return item_class(self, hash_key, range_key, attrs) def query(self, hash_key, *args, **kw): """ Perform a query on the table. :type hash_key: int|long|float|str|unicode|Binary :param hash_key: The HashKey of the requested item. The type of the value must match the type defined in the schema for the table. :type range_key_condition: :class:`boto.dynamodb.condition.Condition` :param range_key_condition: A Condition object. Condition object can be one of the following types: EQ|LE|LT|GE|GT|BEGINS_WITH|BETWEEN The only condition which expects or will accept two values is 'BETWEEN', otherwise a single value should be passed to the Condition constructor. :type attributes_to_get: list :param attributes_to_get: A list of attribute names. If supplied, only the specified attribute names will be returned. Otherwise, all attributes will be returned. :type request_limit: int :param request_limit: The maximum number of items to retrieve from Amazon DynamoDB on each request. You may want to set a specific request_limit based on the provisioned throughput of your table. The default behavior is to retrieve as many results as possible per request. :type max_results: int :param max_results: The maximum number of results that will be retrieved from Amazon DynamoDB in total. For example, if you only wanted to see the first 100 results from the query, regardless of how many were actually available, you could set max_results to 100 and the generator returned from the query method will only yeild 100 results max. :type consistent_read: bool :param consistent_read: If True, a consistent read request is issued. Otherwise, an eventually consistent request is issued. :type scan_index_forward: bool :param scan_index_forward: Specified forward or backward traversal of the index. Default is forward (True). :type exclusive_start_key: list or tuple :param exclusive_start_key: Primary key of the item from which to continue an earlier query. This would be provided as the LastEvaluatedKey in that query. :type count: bool :param count: If True, Amazon DynamoDB returns a total number of items for the Query operation, even if the operation has no matching items for the assigned filter. If count is True, the actual items are not returned and the count is accessible as the ``count`` attribute of the returned object. :type item_class: Class :param item_class: Allows you to override the class used to generate the items. This should be a subclass of :class:`boto.dynamodb.item.Item` """ return self.layer2.query(self, hash_key, *args, **kw) def scan(self, *args, **kw): """ Scan through this table, this is a very long and expensive operation, and should be avoided if at all possible. :type scan_filter: A dict :param scan_filter: A dictionary where the key is the attribute name and the value is a :class:`boto.dynamodb.condition.Condition` object. Valid Condition objects include: * EQ - equal (1) * NE - not equal (1) * LE - less than or equal (1) * LT - less than (1) * GE - greater than or equal (1) * GT - greater than (1) * NOT_NULL - attribute exists (0, use None) * NULL - attribute does not exist (0, use None) * CONTAINS - substring or value in list (1) * NOT_CONTAINS - absence of substring or value in list (1) * BEGINS_WITH - substring prefix (1) * IN - exact match in list (N) * BETWEEN - >= first value, <= second value (2) :type attributes_to_get: list :param attributes_to_get: A list of attribute names. If supplied, only the specified attribute names will be returned. Otherwise, all attributes will be returned. :type request_limit: int :param request_limit: The maximum number of items to retrieve from Amazon DynamoDB on each request. You may want to set a specific request_limit based on the provisioned throughput of your table. The default behavior is to retrieve as many results as possible per request. :type max_results: int :param max_results: The maximum number of results that will be retrieved from Amazon DynamoDB in total. For example, if you only wanted to see the first 100 results from the query, regardless of how many were actually available, you could set max_results to 100 and the generator returned from the query method will only yeild 100 results max. :type count: bool :param count: If True, Amazon DynamoDB returns a total number of items for the Scan operation, even if the operation has no matching items for the assigned filter. If count is True, the actual items are not returned and the count is accessible as the ``count`` attribute of the returned object. :type exclusive_start_key: list or tuple :param exclusive_start_key: Primary key of the item from which to continue an earlier query. This would be provided as the LastEvaluatedKey in that query. :type item_class: Class :param item_class: Allows you to override the class used to generate the items. This should be a subclass of :class:`boto.dynamodb.item.Item` :return: A TableGenerator (generator) object which will iterate over all results :rtype: :class:`boto.dynamodb.layer2.TableGenerator` """ return self.layer2.scan(self, *args, **kw) def batch_get_item(self, keys, attributes_to_get=None): """ Return a set of attributes for a multiple items from a single table using their primary keys. This abstraction removes the 100 Items per batch limitations as well as the "UnprocessedKeys" logic. :type keys: list :param keys: A list of scalar or tuple values. Each element in the list represents one Item to retrieve. If the schema for the table has both a HashKey and a RangeKey, each element in the list should be a tuple consisting of (hash_key, range_key). If the schema for the table contains only a HashKey, each element in the list should be a scalar value of the appropriate type for the table schema. NOTE: The maximum number of items that can be retrieved for a single operation is 100. Also, the number of items retrieved is constrained by a 1 MB size limit. :type attributes_to_get: list :param attributes_to_get: A list of attribute names. If supplied, only the specified attribute names will be returned. Otherwise, all attributes will be returned. :return: A TableBatchGenerator (generator) object which will iterate over all results :rtype: :class:`boto.dynamodb.table.TableBatchGenerator` """ return TableBatchGenerator(self, keys, attributes_to_get)