a1exlism
10/7/2019 - 6:19 AM

large file read with python | Read files with bash commands

large_file_read.py
content_copyfile_download
# LINK https://segmentfault.com/a/1190000005356779
#!/usr/bin/env python
# coding=utf-8
import sys
import time

start = time.time()
lines = 0

for each_line in sys.stdin:
    lines += 1

stop = time.time()

print 'The total lines:', lines, ', it\'s takes', stop - start, 'seconds'

# TIPS bash commands
data du - sh access.log
# 3.1G    access.log
data cat access.log | python filter.py
# The total lines: 11024424, it's takes 87.4551298618 seconds