JONCHAN-CN
4/11/2020 - 9:31 AM

jdbc-hive-threading

# https://zhuanlan.zhihu.com/p/64717840
from concurrent import futures
import pysnooper

MAX_WORKERS = 5

@pysnooper.snoop()
def get_jdbc_connection():
    from subprocess import Popen, PIPE
    import jaydebeapi
    import jpype

    if jpype.isJVMStarted() and not jpype.isThreadAttachedToJVM():
        jpype.attachThreadToJVM()
        jpype.java.lang.Thread.currentThread().setContextClassLoader(jpype.java.lang.ClassLoader.getSystemClassLoader())  

    DATABASE  = 'default'
    DRIVER    = 'org.apache.hive.jdbc.HiveDriver'
    SERVER    = 'localhost'
    KEYTAB    = '/etc/krbX.keytab'
    PRINCIPAL = 'hadoop/xxx@EXAMPLE.COM'
    PORT      = 10000
    JARSPATH  = ["/home/log-analyse/test/jdbc/hive-jdbc-2.1.1.jar"]
    USERNAME  = ''
    PASSWORD  = ''

    # JDBC connection string
    URL = "jdbc:hive2://" + SERVER + ":" + str(PORT) + "/"+ DATABASE +";principal=" + PRINCIPAL + ";"
    # Connect to HiveServer2 
    
    conn = jaydebeapi.connect(DRIVER, URL, [USERNAME, PASSWORD], JARSPATH)
    return conn
    
def mainAction():
    conn = get_jdbc_connection()
    cursor = conn.cursor()
    # Execute SQL query
    sql = "select * from log_test limit 10"
    cursor.execute(sql)
    results = cursor.fetchall()
    cursor.close()
    conn.close()
    return results

if __name__ == '__main__':
    with futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        to_do = executor.submit(mainAction)
        for feture in futures.as_completed([to_do]):
            res = feture.result()
            msg = '{} result: {!r}'
            print(msg.format(feture, res))