10/1/2017 - 3:57 AM

Google Cloud Python - Examples

GCloud python datastore pool example. Original source from official Google Cloud Platform documentation for python: (see site for more detailed info on using python with gcloud).

'''Example usage of datastore population, access, and deletion from
 A combnation of examples found under the datastore folder found via Google on

## Simple example of client Retrieval

from import datastore

client = datastore.Client()
product_key = client.key('Product', 123)


from __future__ import print_function

import os

import six

from import datastore

ANCESTOR = ('Book', 'GoT')

RICKARD = ANCESTOR + ('Character', 'Rickard')

EDDARD = RICKARD + ('Character', 'Eddard')

    ANCESTOR + ('Character', 'Catelyn'),
    EDDARD + ('Character', 'Arya'),
    EDDARD + ('Character', 'Sansa'),
    EDDARD + ('Character', 'Robb'),
    EDDARD + ('Character', 'Bran'),
    EDDARD + ('Character', 'Jon Snow'),

        'name': u'Rickard',
        'family': u'Stark',
        'appearances': 0,
        'alive': False,
    }, {
        'name': u'Eddard',
        'family': u'Stark',
        'appearances': 9,
        'alive': False,
    }, {
        'name': u'Catelyn',
        'family': [u'Stark', u'Tully'],
        'appearances': 26,
        'alive': False,
    }, {
        'name': u'Arya',
        'family': u'Stark',
        'appearances': 33,
        'alive': True,
    }, {
        'name': u'Sansa',
        'family': u'Stark',
        'appearances': 31,
        'alive': True,
    }, {
        'name': u'Robb',
        'family': u'Stark',
        'appearances': 22,
        'alive': False,
    }, {
        'name': u'Bran',
        'family': u'Stark',
        'appearances': 25,
        'alive': True,
    }, {
        'name': u'Jon Snow',
        'family': u'Stark',
        'appearances': 32,
        'alive': True,


def print_func(message):
    if os.getenv('GOOGLE_CLOUD_NO_PRINT') != 'true':

def add_characters(client=None):
    if client is None:
        # Get a client that uses the test dataset.
        client = datastore.Client()
    with client.transaction() as xact:
        for key_path, character in, CHARACTERS):
            if key_path[-1] != character['name']:
                raise ValueError(('Character and key don\'t agree',
                                  key_path, character))
            entity = datastore.Entity(key=client.key(*key_path))
            print_func('Adding Character %s %s' % (character['name'],

if __name__ == '__main__':

# End Populating    

# Begin Clearing

def print_func(message):
    if os.getenv('GOOGLE_CLOUD_NO_PRINT') != 'true':

def fetch_keys(kind, client, fetch_max=FETCH_MAX, query=None, cursor=None):
    if query is None:
        query = client.query(kind=kind)

    iterator = query.fetch(limit=fetch_max, start_cursor=cursor)
    page =
    return query, list(page), iterator.next_page_token

def get_ancestors(entities):
    # NOTE: A key will always have at least one path element.
    key_roots = [entity.key.flat_path[:2] for entity in entities]
    # Return the unique roots.
    return list(set(key_roots))

def remove_kind(kind, client):
    results = []

    query, curr_results, cursor = fetch_keys(kind, client)
    while curr_results:
        query, curr_results, cursor = fetch_keys(
            kind, client, query=query, cursor=cursor)

    if not results:

    delete_outside_transaction = False
    with client.transaction():
        # Now that we have all results, we seek to delete.
        print_func('Deleting keys:')

        ancestors = get_ancestors(results)
        if len(ancestors) > TRANSACTION_MAX_GROUPS:
            delete_outside_transaction = True
            client.delete_multi([result.key for result in results])

    if delete_outside_transaction:
        client.delete_multi([result.key for result in results])

def remove_all_entities(client=None):
    if client is None:
        # Get a client that uses the test dataset.
        client = datastore.Client()
    for kind in ALL_KINDS:
        remove_kind(kind, client)

if __name__ == '__main__':
    print_func('This command will remove all entities for '
               'the following kinds:')
    print_func('\n'.join('- ' + val for val in ALL_KINDS))
    response = six.moves.input('Is this OK [y/n]? ')
    if response.lower() == 'y':
        print_func('Doing nothing.')
# end Clearing

import logging
import threading
import time
import weakref

logger = logging.getLogger(__name__)

class ResourcePool(object):
    """Manage a pool of resourcess.
    There's no limit on the number of resourcess a pool can keep track of,
    but a warning is logged if there are more than ``pool.size`` active
    resources, and a critical problem if more than twice ``pool.size``.
    New resourcess are registered via add().  This will log a message if
    "too many" resourcess are active.
    When a resource is explicitly closed, return it via ``pool.check_in()``.
    That adds the resource to a stack of resources available for
    reuse, and throws away the oldest stack entries if the stack is too
    large.  ``pool.check_out()`` pops this stack.
    When a resource is obtained via ``pool.check_out()``, the pool holds only a
    weak reference to it thereafter.  It's not necessary to inform the pool
    if the resource goes away.  A resource handed out by ``pool.check_out()``
    counts against ``pool.size`` only so long as it exists, and provided it
    isn't returned via ``pool.check_in()``.
    We retain weak references to "checked out" resources  to allow debugging
    / monitoring.

    def __init__(self, size=4, timeout=1<<31, logger=logger):

        self._lock = threading.RLock()
        self._logger = logger
        self._size = size
        self._timeout = timeout  # seconds

        # A weak mapping, id(resource) -> resource.
        self._all = weakref.WeakValueDictionary()

        # A stack of resources available to check out.
        self._available = []

    def __enter__(self):  # pragma: no cover
        return self._lock.__enter__()

    def __exit__(self, etype, err, tb):  # pragma: no cover
        return self._lock.__exit__(etype, err, tb)

    def __iter__(self):
        with self._lock:
            return iter(list(self._all.values()))

    def size(self):
        """Expected maximum # of live resources.
        return self._size

    def size(self, size):
        """Change the expected maximum # of live resources.
        If the new size is smaller than the current value, this may discard
        the oldest available resources.
        with self._lock:
            self._size = size

    def timeout(self):
        """Max # of seconds to keep a resource in the pool.
        return self._timeout

    def timeout(self, timeout):
        """Change the max # of seconds to keep a resource in the pool.
        If the new timeout is smaller than the old value, this may discard
        the oldest available resources.
        with self._lock:
            old, self._timeout = self._timeout, timeout
            if timeout < old:

    def available(self):
        """Return a set of the available connections.
        with self._lock:
            return set([resource for timestamp, resource in self._available])

    def add(self, resource, checked_out=False):
        """Add a new resource to the pool.
        Raise ValueError if ``resource`` is already in the pool.
        If ``checked_out`` is False, push ``resource`` onto the available
        stack even if we're over the pool size limit (but shrink the pool
        if needed, potentially discarding older resources).
        If ``checked_out`` is True, do *not* push the resource onto
        the available stack:  the caller will presumably release it later
        via a call to ``check_in``.
        with self._lock:
            if id(resource) in self._all:
                raise ValueError("Resource already in the pool")
            self._all[id(resource)] = resource
            if not checked_out:
                self._shrink(self.size - 1)
            n = len(self._all)
            if n > self.size:
                reporter = self._logger.warning
                if n > 2 * self.size:
                    reporter = self._logger.critical
                reporter("Pool has %s resources with a size of %s",
                        n, self.size)

    def check_in(self, resource):
        """Release a checked-out resource back to the pool.
        Push ``resource`` onto the stack of available resources.
        May discard older available resources.
        with self._lock:
            if id(resource) not in self._all:
                raise ValueError("Unknown resource:  use 'add()'")
            if resource in self.available:
                raise ValueError("Resource already checked in")
            self._shrink(self.size - 1)

    def check_out(self):
        """Pop an available resource and return it.
        Return None if none are available.  In that case, the caller might
        create a new resource and register it via ``add()``, passing the
        ``checked_out`` flag to retain use of the resource..
        with self._lock:
            if self._available:
                return self._available.pop()[1]

    def _append(self, resource):
        """Push a timestamped resource onto the stack available for checkout.
        Assumes ``self._lock`` is already acquired.
        self._available.append((time.time(), resource))

    def _shrink(self, target):
        """Discard oldest available resources to meet the given target size.
        Assumes ``self._lock`` is already acquired.
        threshhold = time.time() - self.timeout

        available = self._available
        while (len(available) > target or
              available and available[0][0] < threshhold):
            resource = available.pop(0)[1]
            del self._all[id(resource)]