compat / subprocess.pyon commit cvsimport - cleanup of the multi-indexes handling (c5f448b)
   1# subprocess - Subprocesses with accessible I/O streams
   2#
   3# For more information about this module, see PEP 324.
   4#
   5# This module should remain compatible with Python 2.2, see PEP 291.
   6#
   7# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
   8#
   9# Licensed to PSF under a Contributor Agreement.
  10# See http://www.python.org/2.4/license for licensing details.
  11
  12r"""subprocess - Subprocesses with accessible I/O streams
  13
  14This module allows you to spawn processes, connect to their
  15input/output/error pipes, and obtain their return codes.  This module
  16intends to replace several other, older modules and functions, like:
  17
  18os.system
  19os.spawn*
  20os.popen*
  21popen2.*
  22commands.*
  23
  24Information about how the subprocess module can be used to replace these
  25modules and functions can be found below.
  26
  27
  28
  29Using the subprocess module
  30===========================
  31This module defines one class called Popen:
  32
  33class Popen(args, bufsize=0, executable=None,
  34            stdin=None, stdout=None, stderr=None,
  35            preexec_fn=None, close_fds=False, shell=False,
  36            cwd=None, env=None, universal_newlines=False,
  37            startupinfo=None, creationflags=0):
  38
  39
  40Arguments are:
  41
  42args should be a string, or a sequence of program arguments.  The
  43program to execute is normally the first item in the args sequence or
  44string, but can be explicitly set by using the executable argument.
  45
  46On UNIX, with shell=False (default): In this case, the Popen class
  47uses os.execvp() to execute the child program.  args should normally
  48be a sequence.  A string will be treated as a sequence with the string
  49as the only item (the program to execute).
  50
  51On UNIX, with shell=True: If args is a string, it specifies the
  52command string to execute through the shell.  If args is a sequence,
  53the first item specifies the command string, and any additional items
  54will be treated as additional shell arguments.
  55
  56On Windows: the Popen class uses CreateProcess() to execute the child
  57program, which operates on strings.  If args is a sequence, it will be
  58converted to a string using the list2cmdline method.  Please note that
  59not all MS Windows applications interpret the command line the same
  60way: The list2cmdline is designed for applications using the same
  61rules as the MS C runtime.
  62
  63bufsize, if given, has the same meaning as the corresponding argument
  64to the built-in open() function: 0 means unbuffered, 1 means line
  65buffered, any other positive value means use a buffer of
  66(approximately) that size.  A negative bufsize means to use the system
  67default, which usually means fully buffered.  The default value for
  68bufsize is 0 (unbuffered).
  69
  70stdin, stdout and stderr specify the executed programs' standard
  71input, standard output and standard error file handles, respectively.
  72Valid values are PIPE, an existing file descriptor (a positive
  73integer), an existing file object, and None.  PIPE indicates that a
  74new pipe to the child should be created.  With None, no redirection
  75will occur; the child's file handles will be inherited from the
  76parent.  Additionally, stderr can be STDOUT, which indicates that the
  77stderr data from the applications should be captured into the same
  78file handle as for stdout.
  79
  80If preexec_fn is set to a callable object, this object will be called
  81in the child process just before the child is executed.
  82
  83If close_fds is true, all file descriptors except 0, 1 and 2 will be
  84closed before the child process is executed.
  85
  86if shell is true, the specified command will be executed through the
  87shell.
  88
  89If cwd is not None, the current directory will be changed to cwd
  90before the child is executed.
  91
  92If env is not None, it defines the environment variables for the new
  93process.
  94
  95If universal_newlines is true, the file objects stdout and stderr are
  96opened as a text files, but lines may be terminated by any of '\n',
  97the Unix end-of-line convention, '\r', the Macintosh convention or
  98'\r\n', the Windows convention.  All of these external representations
  99are seen as '\n' by the Python program.  Note: This feature is only
 100available if Python is built with universal newline support (the
 101default).  Also, the newlines attribute of the file objects stdout,
 102stdin and stderr are not updated by the communicate() method.
 103
 104The startupinfo and creationflags, if given, will be passed to the
 105underlying CreateProcess() function.  They can specify things such as
 106appearance of the main window and priority for the new process.
 107(Windows only)
 108
 109
 110This module also defines two shortcut functions:
 111
 112call(*args, **kwargs):
 113    Run command with arguments.  Wait for command to complete, then
 114    return the returncode attribute.
 115
 116    The arguments are the same as for the Popen constructor.  Example:
 117
 118    retcode = call(["ls", "-l"])
 119
 120
 121Exceptions
 122----------
 123Exceptions raised in the child process, before the new program has
 124started to execute, will be re-raised in the parent.  Additionally,
 125the exception object will have one extra attribute called
 126'child_traceback', which is a string containing traceback information
 127from the childs point of view.
 128
 129The most common exception raised is OSError.  This occurs, for
 130example, when trying to execute a non-existent file.  Applications
 131should prepare for OSErrors.
 132
 133A ValueError will be raised if Popen is called with invalid arguments.
 134
 135
 136Security
 137--------
 138Unlike some other popen functions, this implementation will never call
 139/bin/sh implicitly.  This means that all characters, including shell
 140metacharacters, can safely be passed to child processes.
 141
 142
 143Popen objects
 144=============
 145Instances of the Popen class have the following methods:
 146
 147poll()
 148    Check if child process has terminated.  Returns returncode
 149    attribute.
 150
 151wait()
 152    Wait for child process to terminate.  Returns returncode attribute.
 153
 154communicate(input=None)
 155    Interact with process: Send data to stdin.  Read data from stdout
 156    and stderr, until end-of-file is reached.  Wait for process to
 157    terminate.  The optional stdin argument should be a string to be
 158    sent to the child process, or None, if no data should be sent to
 159    the child.
 160
 161    communicate() returns a tuple (stdout, stderr).
 162
 163    Note: The data read is buffered in memory, so do not use this
 164    method if the data size is large or unlimited.
 165
 166The following attributes are also available:
 167
 168stdin
 169    If the stdin argument is PIPE, this attribute is a file object
 170    that provides input to the child process.  Otherwise, it is None.
 171
 172stdout
 173    If the stdout argument is PIPE, this attribute is a file object
 174    that provides output from the child process.  Otherwise, it is
 175    None.
 176
 177stderr
 178    If the stderr argument is PIPE, this attribute is file object that
 179    provides error output from the child process.  Otherwise, it is
 180    None.
 181
 182pid
 183    The process ID of the child process.
 184
 185returncode
 186    The child return code.  A None value indicates that the process
 187    hasn't terminated yet.  A negative value -N indicates that the
 188    child was terminated by signal N (UNIX only).
 189
 190
 191Replacing older functions with the subprocess module
 192====================================================
 193In this section, "a ==> b" means that b can be used as a replacement
 194for a.
 195
 196Note: All functions in this section fail (more or less) silently if
 197the executed program cannot be found; this module raises an OSError
 198exception.
 199
 200In the following examples, we assume that the subprocess module is
 201imported with "from subprocess import *".
 202
 203
 204Replacing /bin/sh shell backquote
 205---------------------------------
 206output=`mycmd myarg`
 207==>
 208output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
 209
 210
 211Replacing shell pipe line
 212-------------------------
 213output=`dmesg | grep hda`
 214==>
 215p1 = Popen(["dmesg"], stdout=PIPE)
 216p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
 217output = p2.communicate()[0]
 218
 219
 220Replacing os.system()
 221---------------------
 222sts = os.system("mycmd" + " myarg")
 223==>
 224p = Popen("mycmd" + " myarg", shell=True)
 225sts = os.waitpid(p.pid, 0)
 226
 227Note:
 228
 229* Calling the program through the shell is usually not required.
 230
 231* It's easier to look at the returncode attribute than the
 232  exitstatus.
 233
 234A more real-world example would look like this:
 235
 236try:
 237    retcode = call("mycmd" + " myarg", shell=True)
 238    if retcode < 0:
 239        print >>sys.stderr, "Child was terminated by signal", -retcode
 240    else:
 241        print >>sys.stderr, "Child returned", retcode
 242except OSError, e:
 243    print >>sys.stderr, "Execution failed:", e
 244
 245
 246Replacing os.spawn*
 247-------------------
 248P_NOWAIT example:
 249
 250pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg")
 251==>
 252pid = Popen(["/bin/mycmd", "myarg"]).pid
 253
 254
 255P_WAIT example:
 256
 257retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg")
 258==>
 259retcode = call(["/bin/mycmd", "myarg"])
 260
 261
 262Vector example:
 263
 264os.spawnvp(os.P_NOWAIT, path, args)
 265==>
 266Popen([path] + args[1:])
 267
 268
 269Environment example:
 270
 271os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env)
 272==>
 273Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"})
 274
 275
 276Replacing os.popen*
 277-------------------
 278pipe = os.popen(cmd, mode='r', bufsize)
 279==>
 280pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout
 281
 282pipe = os.popen(cmd, mode='w', bufsize)
 283==>
 284pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin
 285
 286
 287(child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize)
 288==>
 289p = Popen(cmd, shell=True, bufsize=bufsize,
 290          stdin=PIPE, stdout=PIPE, close_fds=True)
 291(child_stdin, child_stdout) = (p.stdin, p.stdout)
 292
 293
 294(child_stdin,
 295 child_stdout,
 296 child_stderr) = os.popen3(cmd, mode, bufsize)
 297==>
 298p = Popen(cmd, shell=True, bufsize=bufsize,
 299          stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
 300(child_stdin,
 301 child_stdout,
 302 child_stderr) = (p.stdin, p.stdout, p.stderr)
 303
 304
 305(child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize)
 306==>
 307p = Popen(cmd, shell=True, bufsize=bufsize,
 308          stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True)
 309(child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout)
 310
 311
 312Replacing popen2.*
 313------------------
 314Note: If the cmd argument to popen2 functions is a string, the command
 315is executed through /bin/sh.  If it is a list, the command is directly
 316executed.
 317
 318(child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode)
 319==>
 320p = Popen(["somestring"], shell=True, bufsize=bufsize
 321          stdin=PIPE, stdout=PIPE, close_fds=True)
 322(child_stdout, child_stdin) = (p.stdout, p.stdin)
 323
 324
 325(child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode)
 326==>
 327p = Popen(["mycmd", "myarg"], bufsize=bufsize,
 328          stdin=PIPE, stdout=PIPE, close_fds=True)
 329(child_stdout, child_stdin) = (p.stdout, p.stdin)
 330
 331The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen,
 332except that:
 333
 334* subprocess.Popen raises an exception if the execution fails
 335* the capturestderr argument is replaced with the stderr argument.
 336* stdin=PIPE and stdout=PIPE must be specified.
 337* popen2 closes all filedescriptors by default, but you have to specify
 338  close_fds=True with subprocess.Popen.
 339
 340
 341"""
 342
 343import sys
 344mswindows = (sys.platform == "win32")
 345
 346import os
 347import types
 348import traceback
 349
 350if mswindows:
 351    import threading
 352    import msvcrt
 353    if 0: # <-- change this to use pywin32 instead of the _subprocess driver
 354        import pywintypes
 355        from win32api import GetStdHandle, STD_INPUT_HANDLE, \
 356                             STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
 357        from win32api import GetCurrentProcess, DuplicateHandle, \
 358                             GetModuleFileName, GetVersion
 359        from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE
 360        from win32pipe import CreatePipe
 361        from win32process import CreateProcess, STARTUPINFO, \
 362                                 GetExitCodeProcess, STARTF_USESTDHANDLES, \
 363                                 STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE
 364        from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
 365    else:
 366        from _subprocess import *
 367        class STARTUPINFO:
 368            dwFlags = 0
 369            hStdInput = None
 370            hStdOutput = None
 371            hStdError = None
 372        class pywintypes:
 373            error = IOError
 374else:
 375    import select
 376    import errno
 377    import fcntl
 378    import pickle
 379
 380__all__ = ["Popen", "PIPE", "STDOUT", "call"]
 381
 382try:
 383    MAXFD = os.sysconf("SC_OPEN_MAX")
 384except:
 385    MAXFD = 256
 386
 387# True/False does not exist on 2.2.0
 388try:
 389    False
 390except NameError:
 391    False = 0
 392    True = 1
 393
 394_active = []
 395
 396def _cleanup():
 397    for inst in _active[:]:
 398        inst.poll()
 399
 400PIPE = -1
 401STDOUT = -2
 402
 403
 404def call(*args, **kwargs):
 405    """Run command with arguments.  Wait for command to complete, then
 406    return the returncode attribute.
 407
 408    The arguments are the same as for the Popen constructor.  Example:
 409
 410    retcode = call(["ls", "-l"])
 411    """
 412    return Popen(*args, **kwargs).wait()
 413
 414
 415def list2cmdline(seq):
 416    """
 417    Translate a sequence of arguments into a command line
 418    string, using the same rules as the MS C runtime:
 419
 420    1) Arguments are delimited by white space, which is either a
 421       space or a tab.
 422
 423    2) A string surrounded by double quotation marks is
 424       interpreted as a single argument, regardless of white space
 425       contained within.  A quoted string can be embedded in an
 426       argument.
 427
 428    3) A double quotation mark preceded by a backslash is
 429       interpreted as a literal double quotation mark.
 430
 431    4) Backslashes are interpreted literally, unless they
 432       immediately precede a double quotation mark.
 433
 434    5) If backslashes immediately precede a double quotation mark,
 435       every pair of backslashes is interpreted as a literal
 436       backslash.  If the number of backslashes is odd, the last
 437       backslash escapes the next double quotation mark as
 438       described in rule 3.
 439    """
 440
 441    # See
 442    # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp
 443    result = []
 444    needquote = False
 445    for arg in seq:
 446        bs_buf = []
 447
 448        # Add a space to separate this argument from the others
 449        if result:
 450            result.append(' ')
 451
 452        needquote = (" " in arg) or ("\t" in arg)
 453        if needquote:
 454            result.append('"')
 455
 456        for c in arg:
 457            if c == '\\':
 458                # Don't know if we need to double yet.
 459                bs_buf.append(c)
 460            elif c == '"':
 461                # Double backspaces.
 462                result.append('\\' * len(bs_buf)*2)
 463                bs_buf = []
 464                result.append('\\"')
 465            else:
 466                # Normal char
 467                if bs_buf:
 468                    result.extend(bs_buf)
 469                    bs_buf = []
 470                result.append(c)
 471
 472        # Add remaining backspaces, if any.
 473        if bs_buf:
 474            result.extend(bs_buf)
 475
 476        if needquote:
 477            result.extend(bs_buf)
 478            result.append('"')
 479
 480    return ''.join(result)
 481
 482
 483class Popen(object):
 484    def __init__(self, args, bufsize=0, executable=None,
 485                 stdin=None, stdout=None, stderr=None,
 486                 preexec_fn=None, close_fds=False, shell=False,
 487                 cwd=None, env=None, universal_newlines=False,
 488                 startupinfo=None, creationflags=0):
 489        """Create new Popen instance."""
 490        _cleanup()
 491
 492        if not isinstance(bufsize, (int, long)):
 493            raise TypeError("bufsize must be an integer")
 494
 495        if mswindows:
 496            if preexec_fn is not None:
 497                raise ValueError("preexec_fn is not supported on Windows "
 498                                 "platforms")
 499            if close_fds:
 500                raise ValueError("close_fds is not supported on Windows "
 501                                 "platforms")
 502        else:
 503            # POSIX
 504            if startupinfo is not None:
 505                raise ValueError("startupinfo is only supported on Windows "
 506                                 "platforms")
 507            if creationflags != 0:
 508                raise ValueError("creationflags is only supported on Windows "
 509                                 "platforms")
 510
 511        self.stdin = None
 512        self.stdout = None
 513        self.stderr = None
 514        self.pid = None
 515        self.returncode = None
 516        self.universal_newlines = universal_newlines
 517
 518        # Input and output objects. The general principle is like
 519        # this:
 520        #
 521        # Parent                   Child
 522        # ------                   -----
 523        # p2cwrite   ---stdin--->  p2cread
 524        # c2pread    <--stdout---  c2pwrite
 525        # errread    <--stderr---  errwrite
 526        #
 527        # On POSIX, the child objects are file descriptors.  On
 528        # Windows, these are Windows file handles.  The parent objects
 529        # are file descriptors on both platforms.  The parent objects
 530        # are None when not using PIPEs. The child objects are None
 531        # when not redirecting.
 532
 533        (p2cread, p2cwrite,
 534         c2pread, c2pwrite,
 535         errread, errwrite) = self._get_handles(stdin, stdout, stderr)
 536
 537        self._execute_child(args, executable, preexec_fn, close_fds,
 538                            cwd, env, universal_newlines,
 539                            startupinfo, creationflags, shell,
 540                            p2cread, p2cwrite,
 541                            c2pread, c2pwrite,
 542                            errread, errwrite)
 543
 544        if p2cwrite:
 545            self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
 546        if c2pread:
 547            if universal_newlines:
 548                self.stdout = os.fdopen(c2pread, 'rU', bufsize)
 549            else:
 550                self.stdout = os.fdopen(c2pread, 'rb', bufsize)
 551        if errread:
 552            if universal_newlines:
 553                self.stderr = os.fdopen(errread, 'rU', bufsize)
 554            else:
 555                self.stderr = os.fdopen(errread, 'rb', bufsize)
 556
 557        _active.append(self)
 558
 559
 560    def _translate_newlines(self, data):
 561        data = data.replace("\r\n", "\n")
 562        data = data.replace("\r", "\n")
 563        return data
 564
 565
 566    if mswindows:
 567        #
 568        # Windows methods
 569        #
 570        def _get_handles(self, stdin, stdout, stderr):
 571            """Construct and return tupel with IO objects:
 572            p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 573            """
 574            if stdin == None and stdout == None and stderr == None:
 575                return (None, None, None, None, None, None)
 576
 577            p2cread, p2cwrite = None, None
 578            c2pread, c2pwrite = None, None
 579            errread, errwrite = None, None
 580
 581            if stdin == None:
 582                p2cread = GetStdHandle(STD_INPUT_HANDLE)
 583            elif stdin == PIPE:
 584                p2cread, p2cwrite = CreatePipe(None, 0)
 585                # Detach and turn into fd
 586                p2cwrite = p2cwrite.Detach()
 587                p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0)
 588            elif type(stdin) == types.IntType:
 589                p2cread = msvcrt.get_osfhandle(stdin)
 590            else:
 591                # Assuming file-like object
 592                p2cread = msvcrt.get_osfhandle(stdin.fileno())
 593            p2cread = self._make_inheritable(p2cread)
 594
 595            if stdout == None:
 596                c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE)
 597            elif stdout == PIPE:
 598                c2pread, c2pwrite = CreatePipe(None, 0)
 599                # Detach and turn into fd
 600                c2pread = c2pread.Detach()
 601                c2pread = msvcrt.open_osfhandle(c2pread, 0)
 602            elif type(stdout) == types.IntType:
 603                c2pwrite = msvcrt.get_osfhandle(stdout)
 604            else:
 605                # Assuming file-like object
 606                c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
 607            c2pwrite = self._make_inheritable(c2pwrite)
 608
 609            if stderr == None:
 610                errwrite = GetStdHandle(STD_ERROR_HANDLE)
 611            elif stderr == PIPE:
 612                errread, errwrite = CreatePipe(None, 0)
 613                # Detach and turn into fd
 614                errread = errread.Detach()
 615                errread = msvcrt.open_osfhandle(errread, 0)
 616            elif stderr == STDOUT:
 617                errwrite = c2pwrite
 618            elif type(stderr) == types.IntType:
 619                errwrite = msvcrt.get_osfhandle(stderr)
 620            else:
 621                # Assuming file-like object
 622                errwrite = msvcrt.get_osfhandle(stderr.fileno())
 623            errwrite = self._make_inheritable(errwrite)
 624
 625            return (p2cread, p2cwrite,
 626                    c2pread, c2pwrite,
 627                    errread, errwrite)
 628
 629
 630        def _make_inheritable(self, handle):
 631            """Return a duplicate of handle, which is inheritable"""
 632            return DuplicateHandle(GetCurrentProcess(), handle,
 633                                   GetCurrentProcess(), 0, 1,
 634                                   DUPLICATE_SAME_ACCESS)
 635
 636
 637        def _find_w9xpopen(self):
 638            """Find and return absolut path to w9xpopen.exe"""
 639            w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)),
 640                                    "w9xpopen.exe")
 641            if not os.path.exists(w9xpopen):
 642                # Eeek - file-not-found - possibly an embedding
 643                # situation - see if we can locate it in sys.exec_prefix
 644                w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix),
 645                                        "w9xpopen.exe")
 646                if not os.path.exists(w9xpopen):
 647                    raise RuntimeError("Cannot locate w9xpopen.exe, which is "
 648                                       "needed for Popen to work with your "
 649                                       "shell or platform.")
 650            return w9xpopen
 651
 652
 653        def _execute_child(self, args, executable, preexec_fn, close_fds,
 654                           cwd, env, universal_newlines,
 655                           startupinfo, creationflags, shell,
 656                           p2cread, p2cwrite,
 657                           c2pread, c2pwrite,
 658                           errread, errwrite):
 659            """Execute program (MS Windows version)"""
 660
 661            if not isinstance(args, types.StringTypes):
 662                args = list2cmdline(args)
 663
 664            # Process startup details
 665            default_startupinfo = STARTUPINFO()
 666            if startupinfo == None:
 667                startupinfo = default_startupinfo
 668            if not None in (p2cread, c2pwrite, errwrite):
 669                startupinfo.dwFlags |= STARTF_USESTDHANDLES
 670                startupinfo.hStdInput = p2cread
 671                startupinfo.hStdOutput = c2pwrite
 672                startupinfo.hStdError = errwrite
 673
 674            if shell:
 675                default_startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 676                default_startupinfo.wShowWindow = SW_HIDE
 677                comspec = os.environ.get("COMSPEC", "cmd.exe")
 678                args = comspec + " /c " + args
 679                if (GetVersion() >= 0x80000000L or
 680                        os.path.basename(comspec).lower() == "command.com"):
 681                    # Win9x, or using command.com on NT. We need to
 682                    # use the w9xpopen intermediate program. For more
 683                    # information, see KB Q150956
 684                    # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp)
 685                    w9xpopen = self._find_w9xpopen()
 686                    args = '"%s" %s' % (w9xpopen, args)
 687                    # Not passing CREATE_NEW_CONSOLE has been known to
 688                    # cause random failures on win9x.  Specifically a
 689                    # dialog: "Your program accessed mem currently in
 690                    # use at xxx" and a hopeful warning about the
 691                    # stability of your system.  Cost is Ctrl+C wont
 692                    # kill children.
 693                    creationflags |= CREATE_NEW_CONSOLE
 694
 695            # Start the process
 696            try:
 697                hp, ht, pid, tid = CreateProcess(executable, args,
 698                                         # no special security
 699                                         None, None,
 700                                         # must inherit handles to pass std
 701                                         # handles
 702                                         1,
 703                                         creationflags,
 704                                         env,
 705                                         cwd,
 706                                         startupinfo)
 707            except pywintypes.error, e:
 708                # Translate pywintypes.error to WindowsError, which is
 709                # a subclass of OSError.  FIXME: We should really
 710                # translate errno using _sys_errlist (or simliar), but
 711                # how can this be done from Python?
 712                raise WindowsError(*e.args)
 713
 714            # Retain the process handle, but close the thread handle
 715            self._handle = hp
 716            self.pid = pid
 717            ht.Close()
 718
 719            # Child is launched. Close the parent's copy of those pipe
 720            # handles that only the child should have open.  You need
 721            # to make sure that no handles to the write end of the
 722            # output pipe are maintained in this process or else the
 723            # pipe will not close when the child process exits and the
 724            # ReadFile will hang.
 725            if p2cread != None:
 726                p2cread.Close()
 727            if c2pwrite != None:
 728                c2pwrite.Close()
 729            if errwrite != None:
 730                errwrite.Close()
 731
 732
 733        def poll(self):
 734            """Check if child process has terminated.  Returns returncode
 735            attribute."""
 736            if self.returncode == None:
 737                if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0:
 738                    self.returncode = GetExitCodeProcess(self._handle)
 739                    _active.remove(self)
 740            return self.returncode
 741
 742
 743        def wait(self):
 744            """Wait for child process to terminate.  Returns returncode
 745            attribute."""
 746            if self.returncode == None:
 747                obj = WaitForSingleObject(self._handle, INFINITE)
 748                self.returncode = GetExitCodeProcess(self._handle)
 749                _active.remove(self)
 750            return self.returncode
 751
 752
 753        def _readerthread(self, fh, buffer):
 754            buffer.append(fh.read())
 755
 756
 757        def communicate(self, input=None):
 758            """Interact with process: Send data to stdin.  Read data from
 759            stdout and stderr, until end-of-file is reached.  Wait for
 760            process to terminate.  The optional input argument should be a
 761            string to be sent to the child process, or None, if no data
 762            should be sent to the child.
 763
 764            communicate() returns a tuple (stdout, stderr)."""
 765            stdout = None # Return
 766            stderr = None # Return
 767
 768            if self.stdout:
 769                stdout = []
 770                stdout_thread = threading.Thread(target=self._readerthread,
 771                                                 args=(self.stdout, stdout))
 772                stdout_thread.setDaemon(True)
 773                stdout_thread.start()
 774            if self.stderr:
 775                stderr = []
 776                stderr_thread = threading.Thread(target=self._readerthread,
 777                                                 args=(self.stderr, stderr))
 778                stderr_thread.setDaemon(True)
 779                stderr_thread.start()
 780
 781            if self.stdin:
 782                if input != None:
 783                    self.stdin.write(input)
 784                self.stdin.close()
 785
 786            if self.stdout:
 787                stdout_thread.join()
 788            if self.stderr:
 789                stderr_thread.join()
 790
 791            # All data exchanged.  Translate lists into strings.
 792            if stdout != None:
 793                stdout = stdout[0]
 794            if stderr != None:
 795                stderr = stderr[0]
 796
 797            # Translate newlines, if requested.  We cannot let the file
 798            # object do the translation: It is based on stdio, which is
 799            # impossible to combine with select (unless forcing no
 800            # buffering).
 801            if self.universal_newlines and hasattr(open, 'newlines'):
 802                if stdout:
 803                    stdout = self._translate_newlines(stdout)
 804                if stderr:
 805                    stderr = self._translate_newlines(stderr)
 806
 807            self.wait()
 808            return (stdout, stderr)
 809
 810    else:
 811        #
 812        # POSIX methods
 813        #
 814        def _get_handles(self, stdin, stdout, stderr):
 815            """Construct and return tupel with IO objects:
 816            p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
 817            """
 818            p2cread, p2cwrite = None, None
 819            c2pread, c2pwrite = None, None
 820            errread, errwrite = None, None
 821
 822            if stdin == None:
 823                pass
 824            elif stdin == PIPE:
 825                p2cread, p2cwrite = os.pipe()
 826            elif type(stdin) == types.IntType:
 827                p2cread = stdin
 828            else:
 829                # Assuming file-like object
 830                p2cread = stdin.fileno()
 831
 832            if stdout == None:
 833                pass
 834            elif stdout == PIPE:
 835                c2pread, c2pwrite = os.pipe()
 836            elif type(stdout) == types.IntType:
 837                c2pwrite = stdout
 838            else:
 839                # Assuming file-like object
 840                c2pwrite = stdout.fileno()
 841
 842            if stderr == None:
 843                pass
 844            elif stderr == PIPE:
 845                errread, errwrite = os.pipe()
 846            elif stderr == STDOUT:
 847                errwrite = c2pwrite
 848            elif type(stderr) == types.IntType:
 849                errwrite = stderr
 850            else:
 851                # Assuming file-like object
 852                errwrite = stderr.fileno()
 853
 854            return (p2cread, p2cwrite,
 855                    c2pread, c2pwrite,
 856                    errread, errwrite)
 857
 858
 859        def _set_cloexec_flag(self, fd):
 860            try:
 861                cloexec_flag = fcntl.FD_CLOEXEC
 862            except AttributeError:
 863                cloexec_flag = 1
 864
 865            old = fcntl.fcntl(fd, fcntl.F_GETFD)
 866            fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag)
 867
 868
 869        def _close_fds(self, but):
 870            for i in range(3, MAXFD):
 871                if i == but:
 872                    continue
 873                try:
 874                    os.close(i)
 875                except:
 876                    pass
 877
 878
 879        def _execute_child(self, args, executable, preexec_fn, close_fds,
 880                           cwd, env, universal_newlines,
 881                           startupinfo, creationflags, shell,
 882                           p2cread, p2cwrite,
 883                           c2pread, c2pwrite,
 884                           errread, errwrite):
 885            """Execute program (POSIX version)"""
 886
 887            if isinstance(args, types.StringTypes):
 888                args = [args]
 889
 890            if shell:
 891                args = ["/bin/sh", "-c"] + args
 892
 893            if executable == None:
 894                executable = args[0]
 895
 896            # For transferring possible exec failure from child to parent
 897            # The first char specifies the exception type: 0 means
 898            # OSError, 1 means some other error.
 899            errpipe_read, errpipe_write = os.pipe()
 900            self._set_cloexec_flag(errpipe_write)
 901
 902            self.pid = os.fork()
 903            if self.pid == 0:
 904                # Child
 905                try:
 906                    # Close parent's pipe ends
 907                    if p2cwrite:
 908                        os.close(p2cwrite)
 909                    if c2pread:
 910                        os.close(c2pread)
 911                    if errread:
 912                        os.close(errread)
 913                    os.close(errpipe_read)
 914
 915                    # Dup fds for child
 916                    if p2cread:
 917                        os.dup2(p2cread, 0)
 918                    if c2pwrite:
 919                        os.dup2(c2pwrite, 1)
 920                    if errwrite:
 921                        os.dup2(errwrite, 2)
 922
 923                    # Close pipe fds.  Make sure we doesn't close the same
 924                    # fd more than once.
 925                    if p2cread:
 926                        os.close(p2cread)
 927                    if c2pwrite and c2pwrite not in (p2cread,):
 928                        os.close(c2pwrite)
 929                    if errwrite and errwrite not in (p2cread, c2pwrite):
 930                        os.close(errwrite)
 931
 932                    # Close all other fds, if asked for
 933                    if close_fds:
 934                        self._close_fds(but=errpipe_write)
 935
 936                    if cwd != None:
 937                        os.chdir(cwd)
 938
 939                    if preexec_fn:
 940                        apply(preexec_fn)
 941
 942                    if env == None:
 943                        os.execvp(executable, args)
 944                    else:
 945                        os.execvpe(executable, args, env)
 946
 947                except:
 948                    exc_type, exc_value, tb = sys.exc_info()
 949                    # Save the traceback and attach it to the exception object
 950                    exc_lines = traceback.format_exception(exc_type,
 951                                                           exc_value,
 952                                                           tb)
 953                    exc_value.child_traceback = ''.join(exc_lines)
 954                    os.write(errpipe_write, pickle.dumps(exc_value))
 955
 956                # This exitcode won't be reported to applications, so it
 957                # really doesn't matter what we return.
 958                os._exit(255)
 959
 960            # Parent
 961            os.close(errpipe_write)
 962            if p2cread and p2cwrite:
 963                os.close(p2cread)
 964            if c2pwrite and c2pread:
 965                os.close(c2pwrite)
 966            if errwrite and errread:
 967                os.close(errwrite)
 968
 969            # Wait for exec to fail or succeed; possibly raising exception
 970            data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB
 971            os.close(errpipe_read)
 972            if data != "":
 973                os.waitpid(self.pid, 0)
 974                child_exception = pickle.loads(data)
 975                raise child_exception
 976
 977
 978        def _handle_exitstatus(self, sts):
 979            if os.WIFSIGNALED(sts):
 980                self.returncode = -os.WTERMSIG(sts)
 981            elif os.WIFEXITED(sts):
 982                self.returncode = os.WEXITSTATUS(sts)
 983            else:
 984                # Should never happen
 985                raise RuntimeError("Unknown child exit status!")
 986
 987            _active.remove(self)
 988
 989
 990        def poll(self):
 991            """Check if child process has terminated.  Returns returncode
 992            attribute."""
 993            if self.returncode == None:
 994                try:
 995                    pid, sts = os.waitpid(self.pid, os.WNOHANG)
 996                    if pid == self.pid:
 997                        self._handle_exitstatus(sts)
 998                except os.error:
 999                    pass
1000            return self.returncode
1001
1002
1003        def wait(self):
1004            """Wait for child process to terminate.  Returns returncode
1005            attribute."""
1006            if self.returncode == None:
1007                pid, sts = os.waitpid(self.pid, 0)
1008                self._handle_exitstatus(sts)
1009            return self.returncode
1010
1011
1012        def communicate(self, input=None):
1013            """Interact with process: Send data to stdin.  Read data from
1014            stdout and stderr, until end-of-file is reached.  Wait for
1015            process to terminate.  The optional input argument should be a
1016            string to be sent to the child process, or None, if no data
1017            should be sent to the child.
1018
1019            communicate() returns a tuple (stdout, stderr)."""
1020            read_set = []
1021            write_set = []
1022            stdout = None # Return
1023            stderr = None # Return
1024
1025            if self.stdin:
1026                # Flush stdio buffer.  This might block, if the user has
1027                # been writing to .stdin in an uncontrolled fashion.
1028                self.stdin.flush()
1029                if input:
1030                    write_set.append(self.stdin)
1031                else:
1032                    self.stdin.close()
1033            if self.stdout:
1034                read_set.append(self.stdout)
1035                stdout = []
1036            if self.stderr:
1037                read_set.append(self.stderr)
1038                stderr = []
1039
1040            while read_set or write_set:
1041                rlist, wlist, xlist = select.select(read_set, write_set, [])
1042
1043                if self.stdin in wlist:
1044                    # When select has indicated that the file is writable,
1045                    # we can write up to PIPE_BUF bytes without risk
1046                    # blocking.  POSIX defines PIPE_BUF >= 512
1047                    bytes_written = os.write(self.stdin.fileno(), input[:512])
1048                    input = input[bytes_written:]
1049                    if not input:
1050                        self.stdin.close()
1051                        write_set.remove(self.stdin)
1052
1053                if self.stdout in rlist:
1054                    data = os.read(self.stdout.fileno(), 1024)
1055                    if data == "":
1056                        self.stdout.close()
1057                        read_set.remove(self.stdout)
1058                    stdout.append(data)
1059
1060                if self.stderr in rlist:
1061                    data = os.read(self.stderr.fileno(), 1024)
1062                    if data == "":
1063                        self.stderr.close()
1064                        read_set.remove(self.stderr)
1065                    stderr.append(data)
1066
1067            # All data exchanged.  Translate lists into strings.
1068            if stdout != None:
1069                stdout = ''.join(stdout)
1070            if stderr != None:
1071                stderr = ''.join(stderr)
1072
1073            # Translate newlines, if requested.  We cannot let the file
1074            # object do the translation: It is based on stdio, which is
1075            # impossible to combine with select (unless forcing no
1076            # buffering).
1077            if self.universal_newlines and hasattr(open, 'newlines'):
1078                if stdout:
1079                    stdout = self._translate_newlines(stdout)
1080                if stderr:
1081                    stderr = self._translate_newlines(stderr)
1082
1083            self.wait()
1084            return (stdout, stderr)
1085
1086
1087def _demo_posix():
1088    #
1089    # Example 1: Simple redirection: Get process list
1090    #
1091    plist = Popen(["ps"], stdout=PIPE).communicate()[0]
1092    print "Process list:"
1093    print plist
1094
1095    #
1096    # Example 2: Change uid before executing child
1097    #
1098    if os.getuid() == 0:
1099        p = Popen(["id"], preexec_fn=lambda: os.setuid(100))
1100        p.wait()
1101
1102    #
1103    # Example 3: Connecting several subprocesses
1104    #
1105    print "Looking for 'hda'..."
1106    p1 = Popen(["dmesg"], stdout=PIPE)
1107    p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
1108    print repr(p2.communicate()[0])
1109
1110    #
1111    # Example 4: Catch execution error
1112    #
1113    print
1114    print "Trying a weird file..."
1115    try:
1116        print Popen(["/this/path/does/not/exist"]).communicate()
1117    except OSError, e:
1118        if e.errno == errno.ENOENT:
1119            print "The file didn't exist.  I thought so..."
1120            print "Child traceback:"
1121            print e.child_traceback
1122        else:
1123            print "Error", e.errno
1124    else:
1125        print >>sys.stderr, "Gosh.  No error."
1126
1127
1128def _demo_windows():
1129    #
1130    # Example 1: Connecting several subprocesses
1131    #
1132    print "Looking for 'PROMPT' in set output..."
1133    p1 = Popen("set", stdout=PIPE, shell=True)
1134    p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE)
1135    print repr(p2.communicate()[0])
1136
1137    #
1138    # Example 2: Simple execution of program
1139    #
1140    print "Executing calc..."
1141    p = Popen("calc")
1142    p.wait()
1143
1144
1145if __name__ == "__main__":
1146    if mswindows:
1147        _demo_windows()
1148    else:
1149        _demo_posix()