add partition
drop partition
import datetime
start = datetime.datetime(2017, 8, 22)
end = datetime.datetime(2017, 9, 3)
now = start
partitions = []
while now <= end:
dt = str(now.year) + str(now.month).zfill(2) + str(now.day).zfill(2)
for h in range(24):
partitions.append("drop partition (dt = {dt}, hour = {h});".format(dt=dt, h=h))
now += datetime.timedelta(1)
for partition in partitions:
print(partition)
cursor.execute("""
alter table sugar.conv
{}
""".format(partition)
)
cursor.fetchall()
# add partition
import datetime
start = datetime.datetime(2017, 8, 22)
end = datetime.datetime(2017, 9, 3)
now = start
partitions = []
while now <= end:
dt = str(now.year) + str(now.month).zfill(2) + str(now.day).zfill(2)
for h in range(24):
partitions.append("partition (dt = {dt}, hour = {h}) location 's3://files.sugar/data/raw/sugar_conv/{dt}/{hh}'".format(dt=dt, h=h, hh=str(h).zfill(2)))
now += datetime.timedelta(1)
cursor.execute("""
alter table sugar.conv add
{}
""".format('\n'.join(partitions))
)
cursor.fetchall()