Tail a continuously growing file (like tail -f filename) (Python recipe) by Jason Morriss

ActiveState Code (http://code.activestate.com/recipes/577710/)

""" $Id: filetail.py 1512 2011-05-20 16:14:29Z morrissj $

Python3 module for tailing a file such as a system log that grows continuously.
Transparently handles files that get rotated or trucated.
Inspired by the Perl File::Tail module.

A simple algorithm is used to dynamically sleep when no new data is available in
the file. The longer the amount of time goes by w/o new data the longer the
sleep interval will be (up to "max_interval") and starts at "interval".

Example:
    from filetail import FileTail
    tail = FileTail("/var/log/syslog")
    for line in tail:
        # do something
        pass

"""

import os
import sys
from stat import *
from math import floor
from time import sleep, time

class FileTail(object):
    """
    Tail a file, even if its rotated/truncated.
    Inspiration came from the perl module File::Tail.
    """

    def __init__(self,
                 file,                  # filename to monitor
                 start_pos="end",       # where to initially start reading from
                 #max_buffer_size=16384, # Max buffer size hint (Not exact; @see file.readlines)
                 interval=0.1,          # sleep time to wait if no data is present (dynamically changes)
                 #min_interval=0.01,     # min sleep time
                 max_interval=5,        # max sleep time 
                 max_wait=60,           # max time to wait with no data before reopening file
                 reopen_check="inode",  # how to check if file is different (inode or time) - inode does not work on win32
                 encoding="utf-8"       # file encoding
                ):

        self.start_pos = start_pos
        self.reopen_check = reopen_check
        self.max_wait = max_wait
        #self.max_buffer_size = max_buffer_size
        #self.min_interval = min_interval
        self.max_interval = max_interval
        self.interval = interval
        if self.interval > self.max_interval:
            self.interval = self.max_interval
        self.encoding = encoding

        # will throw exception if it fails... caller should intercept
        self.open(file, start_pos=start_pos)

        # initialize some internal vars
        self._buffer = []
        self.last_time = time()
        self.last_count = 0

    def open(self, file, start_pos="head"):
        """Open the file to tail and initialize our state."""
        fh = open(file, "r", encoding=self.encoding)

        # seek to the initial position in the file we want to start reading
        if start_pos == "end" or start_pos == "tail":
            fh.seek(0, os.SEEK_END)                       # End of file
        elif start_pos == "start" or start_pos == "head":
            #fh.seek(0, os.SEEK_SET)                      # Beginning of file
            pass
        elif start_pos is not None:
            if start_pos >= 0:                            # Absolute position
                fh.seek(start_pos, os.SEEK_SET)
            else:                                         # Absolute position (from end)
                fh.seek(abs(start_pos), os.SEEK_END)
        
        # if we passed the end of the file rewind to the actual end.
        # This avoids a potential race condition if the file was being rotated
        # in the process of opening the file. Not sure if this can actually
        # happen, but better safe than sorry.
        pos = fh.tell()
        if pos > os.stat(file)[ST_SIZE]:
            pos = fh.tell()

        self.fh = fh
        self.pos = pos
        self.stat = os.fstat(fh.fileno())
        self.file = file
    
    def reopen(self):
        """
        Attempt to reopen the current file. If it doesn't appear to have
        changed (been rotated) then the current file handle is not changed.
        """

        #print("Reopening", self.file, "...", end="")

        # if we don't have an opened file already then try to open it now
        if not self.fh or self.fh.closed:
            try:
                self.open(self.file, start_pos="head");
            except IOError:
                return False
            return True

        # save current values
        fh = self.fh
        pos = self.pos
        cur = self.stat
        
        # reopen same file
        try:
            self.open(self.file, "head")
        except IOError as e:
            #print("FILE DOES NOT EXIST")
            return False
        
        new = self.stat
        #print(new.st_ino, ' == ', cur.st_ino)
        if (
            (self.reopen_check == 'inode' and new.st_ino == cur.st_ino)
            or
            (self.reopen_check == 'time' and new.st_mtime <= floor(self.last_time) and new.st_size == pos)
           ):
            #print("FILE NOT CHANGED")
            # file appears to be the same or older than our last read
            #self.last_time = new.st_mtime
            self.fh = fh
            self.pos = pos
            self.stat = cur
            return False

        #print("NEW FILE")
        return True

       
    def __iter__(self):
        """
            Return iterator to support:
                for line in filetail:
                    print line
        """
        self.wait_count = 0
        return self


    def __next__(self):
        """Interator "next" call."""
        return self.next()


    def next(self):
        line = None
        self.wait_count = 0

        # low CPU (probably same as the block below this, but ALLOWS tell()!
        while not line:
            line = self.fh.readline()
            if line != "":
                # track the time we received new data and how much
                self.last_time = time()
                self.last_count = 1
            else:
                self.wait()

        ## uses the least amount of CPU, but does not allow me to tell()
        ## is that a bug in readlines()?
        #while len(self._buffer) == 0:
        #    self._buffer = self.fh.readlines(self.max_buffer_size)
        #    if len(self._buffer) > 0:
        #        # track the time we received new data and how much
        #        self.last_time = time()
        #        self.last_count = len(self._buffer)
        #        self.wait_count = 0
        #    else:
        #        self.wait()
        #line = self._buffer.pop(0)

        # dealing with the file as binary isn't working as well as i hoped
        #while len(self.lines) == 0:
        #    buffer = self.fh.read(self.max_buffer_size).decode(self.encoding)
        #    if buffer is not None:
        #        self._buffer += buffer
        #        size = self.enqueue(self._buffer)
        #        if size:
        #            self._buffer = self._buffer[size:]
        #    else:
        #        self.wait()
        #line = self.lines.pop(0)
            
        # uses too much CPU!! (but not 100%)
        #line = self.fh.readline()
        #while line == "":
        #    self.wait()
        #    line = self.fh.readline()
        #    if line != "":
        #        # track the time we received new data and how much
        #        self.pos = self.fh.tell()
        #        self.last_time = time()
        #        self.last_count = 1 #len(self._buffer)
        #        self.wait_count = 0

        return line

    #def enqueue(self, buffer):
    #    """
    #    Extract any lines from buffer and add to our self.lines list. Ignores
    #    the last line if it does not have a line termination ("\n")
    #    @return total characters extracted from buffer.
    #    """
    #    lines = buffer.splitlines(True)
    #    total = 0;
    #    for l in lines:
    #        if l.endswith("\n"):
    #            self.lines.append(l)
    #            total += len(l)
    #    return total

    # wait for X seconds. The sleep interval is dynamically predicted based on
    # how much was previously read. The predicted interval will never be more
    # than max_interval. If enough time passes w/o any new data the file will
    # be reopened and checked.
    def wait(self):
        if self.wait_count == 0:
            self.pos = self.fh.tell()
            self.stat = os.fstat(self.fh.fileno())

        self.wait_count += 1
        elapsed = time() - self.last_time

        # if we've waited long enough try to reopen the file, if that returns
        # true then we're done here and we do not sleep.
        if elapsed >= self.max_wait:
            self.last_time = time()
            if self.reopen():
                return
            

        # determine delay value. Delay is longer based on total time passed
        # note: currently last_count is always 1.
        if self.last_count:
            #delay = (time() - self.last_time) / self.last_count
            delay = elapsed
        else:
            delay = self.interval

        # don't delay too long
        if delay > self.max_interval:
            delay = self.max_interval
        #elif delay < self.min_interval:
        #    delay = self.min_interval

        #print("delay={:0.06f} elapsed={:0.06f}".format(delay, elapsed))
        sleep(delay)
    
# end of FileTail class


def main():
    print("No tests implemented.")
    
    
if __name__ == "__main__":
    sys.exit(main())