Even with unbuffered stdout/stderr, 'smt run' buffers so we can't win.

Author: brettviren

femb_python/commands.py

@@ -1,16 +1,12 @@
-#!/usr/bin/env python3
 
 import os
-from subprocess import Popen, PIPE, STDOUT
+import sys
+from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
 
-def dump(oline, eline):
-    if oline:
-        print ("Output: %s" % oline)
-    if eline:
-        print ("Output: %s" % eline)
+def noop(text):
+    pass
 
-
-def execute(cmdstr, cwd = ".", env = os.environ, out = dump, shell=True):
+def execute(cmdstr, outcb=noop, errcb=noop, cwd = ".", env = os.environ, shell=True):
     '''
     Execute the given command string.  Return the exit code.
 
@@ -19,10 +15,9 @@ def execute(cmdstr, cwd = ".", env = os.environ, out = dump, shell=True):
     If "env" is given it is used as an environment dictionary for the
     execution, o.w. os.environ will be used.
 
-    If "out" is given it will be called once with either a line of
-    output or a line of error or both.  It may be called multiple
-    times as the process runs.  Any trailing newline on a line is
-    stripped.
+    If "out" is a callable taking two arguments which may be None or
+    text.  It is called per poll of the running command if any text
+    was produced on stdout/stderr.  
 
     If "shell" is True then the cmdstr is passed to the shell.
 
@@ -34,48 +29,29 @@ def execute(cmdstr, cwd = ".", env = os.environ, out = dump, shell=True):
     if not shell and type(cmdstr) == type(""):
         cmdstr = cmdstr.split()
 
-    def slurp(oline, eline):                   # send line to all outs
-        if oline and oline[-1] == '\n':
-            oline = oline[:-1]
-        if eline and eline[-1] == '\n':
-            eline = eline[:-1]
-        out(oline, eline)
-        return
-
-    print('Executing: %s' % cmdstr)
     try:
-        proc = Popen(cmdstr, stdout=PIPE, stderr=PIPE, shell=shell, cwd=cwd,
-                     universal_newlines=True, env=env)
+        proc = Popen(cmdstr, stdout=PIPE, stderr=PIPE,
+                         shell=shell, cwd=cwd, env=env,
+                         bufsize=1, universal_newlines=True)   # line buffered
     except OSError as err:
         print (err)
         raise
 
-    
-    # While command is polled as running, slurp its output and marshal
-    # it to out() until command finishes
-    res = None
     while True:
-        oline = proc.stdout.readline()
-        eline = proc.stderr.readline()
-        res = proc.poll()
-
-        if oline or eline:
-            slurp(oline, eline)
-
-        if res is None:         # still running
+        try:
+            o,e = proc.communicate(timeout=1)
+            outcb(o)
+            errcb(e)
+        except TimeoutExpired:
+            pass
+
+        if proc.returncode is None:
             continue
-
-        for line in proc.stdout.readlines():
-            slurp(line,None)                     # drain any remaining
-        for line in proc.stderr.readlines():
-            slurp(None,line)                     # drain any remaining
-
         break
-
-    if res == 0: 
-        print('Command: "%s" succeeded' % cmdstr)
+        
+    if proc.returncode == 0: 
         return proc.returncode
-    err = 'Command: "%s" failed with code %d in directory %s' % (cmdstr, res, cwd)
+    err = 'Command: "%s" failed with code %d in directory %s' % (cmdstr, proc.returncode, cwd)
     print (err)
     raise RuntimeError( err )
 

femb_python/runpolicy.py

@@ -7,6 +7,7 @@
 
 '''
 import os
+import sys
 import json
 import time
 #import subprocess
@@ -113,15 +114,18 @@ def exec(self, cmdstr, cwd=".", shell=True):
 
         stdout = list()
         stderr = list()
-
-        def out(oline, eline):
-            if oline:
-                stdout.append(oline)
-                print ("Output: %s" % oline)
-            if eline:
-                stderr.append(eline)
-                print ("Error: %s" % eline)
-        rc = commands.execute(cmdstr, shell=shell, cwd=cwd, out=out)
+        def output(text):
+            if not text: return
+            stdout.append(text)
+            sys.stdout.write("Output: " + text)
+            sys.stdout.flush()
+        def errput(text):
+            if not text: return
+            stderr.append(text)
+            sys.stderr.write("Error: " + text)
+
+        rc = commands.execute(cmdstr, output, errput,
+                                  shell=shell, cwd=cwd)
 
         stdout = '\n'.join(stdout)
         stderr = '\n'.join(stderr)

test/live-spew.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# This spews stuff expecting to get live output for testing under a
+# Sumatra runner.
+
+dofail=${1}
+
+echo "live-spew $@"
+
+for n in {0..3}
+do
+    date
+    echo "message number $n to stdout"
+    echo "message number $n to stderr" 1>&2
+    sleep 1
+done
+
+if [ "$dofail" = "fail" ] ; then
+    echo "live-spew.sh will fail now"
+    echo "live-spew.sh will fail now" 1>&2
+    exit 1
+fi
+
+exit 0

test/test_runpolicy.py

@@ -116,13 +116,37 @@ def test_fail():
     else:
         raise RuntimeError("test of /bin/false failed to fail")
 
+def test_live():
+    clear()
+    testdir = os.path.dirname(os.path.realpath(__file__))
+    prog = os.path.join(testdir, "live-spew.sh")
+    r = make_runner(SumatraRunner, smtname="test_live",
+                        executable=prog, argstr="",
+                        #stdout="/tmp/out1.log", stderr="/tmp/err1.log"
+                        )
+    r()
 
+    f = make_runner(SumatraRunner, smtname="test_live",
+                        executable=prog, argstr="fail",
+                        stdout="/dev/stdout", stderr="/dev/stderr"
+                        #stdout="/tmp/out2.log", stderr="/tmp/err2.log"
+                        )
+    try:
+        f()
+    except RuntimeError:
+        print ("Test of failure successfully failed.")
+        pass
+    else:
+        raise RuntimeError("test of failure failed to fail")
+    
+    
 
 if '__main__' == __name__:
+
     test_basic()
     test_direct()
     test_default_args_only()
     test_per_call_args()
     test_with_sumatra()
     test_fail()
-    
+    test_live()