git-p4.pyon commit Merge branch 'sn/null-pointer-arith-in-mark-tree-uninteresting' (782ca8c)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10import sys
  11if sys.hexversion < 0x02040000:
  12    # The limiter is the subprocess module
  13    sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  14    sys.exit(1)
  15import os
  16import optparse
  17import marshal
  18import subprocess
  19import tempfile
  20import time
  21import platform
  22import re
  23import shutil
  24import stat
  25import zipfile
  26import zlib
  27import ctypes
  28
  29try:
  30    from subprocess import CalledProcessError
  31except ImportError:
  32    # from python2.7:subprocess.py
  33    # Exception classes used by this module.
  34    class CalledProcessError(Exception):
  35        """This exception is raised when a process run by check_call() returns
  36        a non-zero exit status.  The exit status will be stored in the
  37        returncode attribute."""
  38        def __init__(self, returncode, cmd):
  39            self.returncode = returncode
  40            self.cmd = cmd
  41        def __str__(self):
  42            return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
  43
  44verbose = False
  45
  46# Only labels/tags matching this will be imported/exported
  47defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  48
  49# Grab changes in blocks of this many revisions, unless otherwise requested
  50defaultBlockSize = 512
  51
  52def p4_build_cmd(cmd):
  53    """Build a suitable p4 command line.
  54
  55    This consolidates building and returning a p4 command line into one
  56    location. It means that hooking into the environment, or other configuration
  57    can be done more easily.
  58    """
  59    real_cmd = ["p4"]
  60
  61    user = gitConfig("git-p4.user")
  62    if len(user) > 0:
  63        real_cmd += ["-u",user]
  64
  65    password = gitConfig("git-p4.password")
  66    if len(password) > 0:
  67        real_cmd += ["-P", password]
  68
  69    port = gitConfig("git-p4.port")
  70    if len(port) > 0:
  71        real_cmd += ["-p", port]
  72
  73    host = gitConfig("git-p4.host")
  74    if len(host) > 0:
  75        real_cmd += ["-H", host]
  76
  77    client = gitConfig("git-p4.client")
  78    if len(client) > 0:
  79        real_cmd += ["-c", client]
  80
  81
  82    if isinstance(cmd,basestring):
  83        real_cmd = ' '.join(real_cmd) + ' ' + cmd
  84    else:
  85        real_cmd += cmd
  86    return real_cmd
  87
  88def chdir(path, is_client_path=False):
  89    """Do chdir to the given path, and set the PWD environment
  90       variable for use by P4.  It does not look at getcwd() output.
  91       Since we're not using the shell, it is necessary to set the
  92       PWD environment variable explicitly.
  93
  94       Normally, expand the path to force it to be absolute.  This
  95       addresses the use of relative path names inside P4 settings,
  96       e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
  97       as given; it looks for .p4config using PWD.
  98
  99       If is_client_path, the path was handed to us directly by p4,
 100       and may be a symbolic link.  Do not call os.getcwd() in this
 101       case, because it will cause p4 to think that PWD is not inside
 102       the client path.
 103       """
 104
 105    os.chdir(path)
 106    if not is_client_path:
 107        path = os.getcwd()
 108    os.environ['PWD'] = path
 109
 110def calcDiskFree():
 111    """Return free space in bytes on the disk of the given dirname."""
 112    if platform.system() == 'Windows':
 113        free_bytes = ctypes.c_ulonglong(0)
 114        ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
 115        return free_bytes.value
 116    else:
 117        st = os.statvfs(os.getcwd())
 118        return st.f_bavail * st.f_frsize
 119
 120def die(msg):
 121    if verbose:
 122        raise Exception(msg)
 123    else:
 124        sys.stderr.write(msg + "\n")
 125        sys.exit(1)
 126
 127def write_pipe(c, stdin):
 128    if verbose:
 129        sys.stderr.write('Writing pipe: %s\n' % str(c))
 130
 131    expand = isinstance(c,basestring)
 132    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
 133    pipe = p.stdin
 134    val = pipe.write(stdin)
 135    pipe.close()
 136    if p.wait():
 137        die('Command failed: %s' % str(c))
 138
 139    return val
 140
 141def p4_write_pipe(c, stdin):
 142    real_cmd = p4_build_cmd(c)
 143    return write_pipe(real_cmd, stdin)
 144
 145def read_pipe(c, ignore_error=False):
 146    if verbose:
 147        sys.stderr.write('Reading pipe: %s\n' % str(c))
 148
 149    expand = isinstance(c,basestring)
 150    p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
 151    (out, err) = p.communicate()
 152    if p.returncode != 0 and not ignore_error:
 153        die('Command failed: %s\nError: %s' % (str(c), err))
 154    return out
 155
 156def p4_read_pipe(c, ignore_error=False):
 157    real_cmd = p4_build_cmd(c)
 158    return read_pipe(real_cmd, ignore_error)
 159
 160def read_pipe_lines(c):
 161    if verbose:
 162        sys.stderr.write('Reading pipe: %s\n' % str(c))
 163
 164    expand = isinstance(c, basestring)
 165    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 166    pipe = p.stdout
 167    val = pipe.readlines()
 168    if pipe.close() or p.wait():
 169        die('Command failed: %s' % str(c))
 170
 171    return val
 172
 173def p4_read_pipe_lines(c):
 174    """Specifically invoke p4 on the command supplied. """
 175    real_cmd = p4_build_cmd(c)
 176    return read_pipe_lines(real_cmd)
 177
 178def p4_has_command(cmd):
 179    """Ask p4 for help on this command.  If it returns an error, the
 180       command does not exist in this version of p4."""
 181    real_cmd = p4_build_cmd(["help", cmd])
 182    p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 183                                   stderr=subprocess.PIPE)
 184    p.communicate()
 185    return p.returncode == 0
 186
 187def p4_has_move_command():
 188    """See if the move command exists, that it supports -k, and that
 189       it has not been administratively disabled.  The arguments
 190       must be correct, but the filenames do not have to exist.  Use
 191       ones with wildcards so even if they exist, it will fail."""
 192
 193    if not p4_has_command("move"):
 194        return False
 195    cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 196    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 197    (out, err) = p.communicate()
 198    # return code will be 1 in either case
 199    if err.find("Invalid option") >= 0:
 200        return False
 201    if err.find("disabled") >= 0:
 202        return False
 203    # assume it failed because @... was invalid changelist
 204    return True
 205
 206def system(cmd, ignore_error=False):
 207    expand = isinstance(cmd,basestring)
 208    if verbose:
 209        sys.stderr.write("executing %s\n" % str(cmd))
 210    retcode = subprocess.call(cmd, shell=expand)
 211    if retcode and not ignore_error:
 212        raise CalledProcessError(retcode, cmd)
 213
 214    return retcode
 215
 216def p4_system(cmd):
 217    """Specifically invoke p4 as the system command. """
 218    real_cmd = p4_build_cmd(cmd)
 219    expand = isinstance(real_cmd, basestring)
 220    retcode = subprocess.call(real_cmd, shell=expand)
 221    if retcode:
 222        raise CalledProcessError(retcode, real_cmd)
 223
 224_p4_version_string = None
 225def p4_version_string():
 226    """Read the version string, showing just the last line, which
 227       hopefully is the interesting version bit.
 228
 229       $ p4 -V
 230       Perforce - The Fast Software Configuration Management System.
 231       Copyright 1995-2011 Perforce Software.  All rights reserved.
 232       Rev. P4/NTX86/2011.1/393975 (2011/12/16).
 233    """
 234    global _p4_version_string
 235    if not _p4_version_string:
 236        a = p4_read_pipe_lines(["-V"])
 237        _p4_version_string = a[-1].rstrip()
 238    return _p4_version_string
 239
 240def p4_integrate(src, dest):
 241    p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 242
 243def p4_sync(f, *options):
 244    p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 245
 246def p4_add(f):
 247    # forcibly add file names with wildcards
 248    if wildcard_present(f):
 249        p4_system(["add", "-f", f])
 250    else:
 251        p4_system(["add", f])
 252
 253def p4_delete(f):
 254    p4_system(["delete", wildcard_encode(f)])
 255
 256def p4_edit(f):
 257    p4_system(["edit", wildcard_encode(f)])
 258
 259def p4_revert(f):
 260    p4_system(["revert", wildcard_encode(f)])
 261
 262def p4_reopen(type, f):
 263    p4_system(["reopen", "-t", type, wildcard_encode(f)])
 264
 265def p4_move(src, dest):
 266    p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 267
 268def p4_last_change():
 269    results = p4CmdList(["changes", "-m", "1"])
 270    return int(results[0]['change'])
 271
 272def p4_describe(change):
 273    """Make sure it returns a valid result by checking for
 274       the presence of field "time".  Return a dict of the
 275       results."""
 276
 277    ds = p4CmdList(["describe", "-s", str(change)])
 278    if len(ds) != 1:
 279        die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 280
 281    d = ds[0]
 282
 283    if "p4ExitCode" in d:
 284        die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 285                                                      str(d)))
 286    if "code" in d:
 287        if d["code"] == "error":
 288            die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 289
 290    if "time" not in d:
 291        die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 292
 293    return d
 294
 295#
 296# Canonicalize the p4 type and return a tuple of the
 297# base type, plus any modifiers.  See "p4 help filetypes"
 298# for a list and explanation.
 299#
 300def split_p4_type(p4type):
 301
 302    p4_filetypes_historical = {
 303        "ctempobj": "binary+Sw",
 304        "ctext": "text+C",
 305        "cxtext": "text+Cx",
 306        "ktext": "text+k",
 307        "kxtext": "text+kx",
 308        "ltext": "text+F",
 309        "tempobj": "binary+FSw",
 310        "ubinary": "binary+F",
 311        "uresource": "resource+F",
 312        "uxbinary": "binary+Fx",
 313        "xbinary": "binary+x",
 314        "xltext": "text+Fx",
 315        "xtempobj": "binary+Swx",
 316        "xtext": "text+x",
 317        "xunicode": "unicode+x",
 318        "xutf16": "utf16+x",
 319    }
 320    if p4type in p4_filetypes_historical:
 321        p4type = p4_filetypes_historical[p4type]
 322    mods = ""
 323    s = p4type.split("+")
 324    base = s[0]
 325    mods = ""
 326    if len(s) > 1:
 327        mods = s[1]
 328    return (base, mods)
 329
 330#
 331# return the raw p4 type of a file (text, text+ko, etc)
 332#
 333def p4_type(f):
 334    results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
 335    return results[0]['headType']
 336
 337#
 338# Given a type base and modifier, return a regexp matching
 339# the keywords that can be expanded in the file
 340#
 341def p4_keywords_regexp_for_type(base, type_mods):
 342    if base in ("text", "unicode", "binary"):
 343        kwords = None
 344        if "ko" in type_mods:
 345            kwords = 'Id|Header'
 346        elif "k" in type_mods:
 347            kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 348        else:
 349            return None
 350        pattern = r"""
 351            \$              # Starts with a dollar, followed by...
 352            (%s)            # one of the keywords, followed by...
 353            (:[^$\n]+)?     # possibly an old expansion, followed by...
 354            \$              # another dollar
 355            """ % kwords
 356        return pattern
 357    else:
 358        return None
 359
 360#
 361# Given a file, return a regexp matching the possible
 362# RCS keywords that will be expanded, or None for files
 363# with kw expansion turned off.
 364#
 365def p4_keywords_regexp_for_file(file):
 366    if not os.path.exists(file):
 367        return None
 368    else:
 369        (type_base, type_mods) = split_p4_type(p4_type(file))
 370        return p4_keywords_regexp_for_type(type_base, type_mods)
 371
 372def setP4ExecBit(file, mode):
 373    # Reopens an already open file and changes the execute bit to match
 374    # the execute bit setting in the passed in mode.
 375
 376    p4Type = "+x"
 377
 378    if not isModeExec(mode):
 379        p4Type = getP4OpenedType(file)
 380        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 381        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 382        if p4Type[-1] == "+":
 383            p4Type = p4Type[0:-1]
 384
 385    p4_reopen(p4Type, file)
 386
 387def getP4OpenedType(file):
 388    # Returns the perforce file type for the given file.
 389
 390    result = p4_read_pipe(["opened", wildcard_encode(file)])
 391    match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
 392    if match:
 393        return match.group(1)
 394    else:
 395        die("Could not determine file type for %s (result: '%s')" % (file, result))
 396
 397# Return the set of all p4 labels
 398def getP4Labels(depotPaths):
 399    labels = set()
 400    if isinstance(depotPaths,basestring):
 401        depotPaths = [depotPaths]
 402
 403    for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 404        label = l['label']
 405        labels.add(label)
 406
 407    return labels
 408
 409# Return the set of all git tags
 410def getGitTags():
 411    gitTags = set()
 412    for line in read_pipe_lines(["git", "tag"]):
 413        tag = line.strip()
 414        gitTags.add(tag)
 415    return gitTags
 416
 417def diffTreePattern():
 418    # This is a simple generator for the diff tree regex pattern. This could be
 419    # a class variable if this and parseDiffTreeEntry were a part of a class.
 420    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 421    while True:
 422        yield pattern
 423
 424def parseDiffTreeEntry(entry):
 425    """Parses a single diff tree entry into its component elements.
 426
 427    See git-diff-tree(1) manpage for details about the format of the diff
 428    output. This method returns a dictionary with the following elements:
 429
 430    src_mode - The mode of the source file
 431    dst_mode - The mode of the destination file
 432    src_sha1 - The sha1 for the source file
 433    dst_sha1 - The sha1 fr the destination file
 434    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 435    status_score - The score for the status (applicable for 'C' and 'R'
 436                   statuses). This is None if there is no score.
 437    src - The path for the source file.
 438    dst - The path for the destination file. This is only present for
 439          copy or renames. If it is not present, this is None.
 440
 441    If the pattern is not matched, None is returned."""
 442
 443    match = diffTreePattern().next().match(entry)
 444    if match:
 445        return {
 446            'src_mode': match.group(1),
 447            'dst_mode': match.group(2),
 448            'src_sha1': match.group(3),
 449            'dst_sha1': match.group(4),
 450            'status': match.group(5),
 451            'status_score': match.group(6),
 452            'src': match.group(7),
 453            'dst': match.group(10)
 454        }
 455    return None
 456
 457def isModeExec(mode):
 458    # Returns True if the given git mode represents an executable file,
 459    # otherwise False.
 460    return mode[-3:] == "755"
 461
 462def isModeExecChanged(src_mode, dst_mode):
 463    return isModeExec(src_mode) != isModeExec(dst_mode)
 464
 465def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 466
 467    if isinstance(cmd,basestring):
 468        cmd = "-G " + cmd
 469        expand = True
 470    else:
 471        cmd = ["-G"] + cmd
 472        expand = False
 473
 474    cmd = p4_build_cmd(cmd)
 475    if verbose:
 476        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 477
 478    # Use a temporary file to avoid deadlocks without
 479    # subprocess.communicate(), which would put another copy
 480    # of stdout into memory.
 481    stdin_file = None
 482    if stdin is not None:
 483        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 484        if isinstance(stdin,basestring):
 485            stdin_file.write(stdin)
 486        else:
 487            for i in stdin:
 488                stdin_file.write(i + '\n')
 489        stdin_file.flush()
 490        stdin_file.seek(0)
 491
 492    p4 = subprocess.Popen(cmd,
 493                          shell=expand,
 494                          stdin=stdin_file,
 495                          stdout=subprocess.PIPE)
 496
 497    result = []
 498    try:
 499        while True:
 500            entry = marshal.load(p4.stdout)
 501            if cb is not None:
 502                cb(entry)
 503            else:
 504                result.append(entry)
 505    except EOFError:
 506        pass
 507    exitCode = p4.wait()
 508    if exitCode != 0:
 509        entry = {}
 510        entry["p4ExitCode"] = exitCode
 511        result.append(entry)
 512
 513    return result
 514
 515def p4Cmd(cmd):
 516    list = p4CmdList(cmd)
 517    result = {}
 518    for entry in list:
 519        result.update(entry)
 520    return result;
 521
 522def p4Where(depotPath):
 523    if not depotPath.endswith("/"):
 524        depotPath += "/"
 525    depotPathLong = depotPath + "..."
 526    outputList = p4CmdList(["where", depotPathLong])
 527    output = None
 528    for entry in outputList:
 529        if "depotFile" in entry:
 530            # Search for the base client side depot path, as long as it starts with the branch's P4 path.
 531            # The base path always ends with "/...".
 532            if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
 533                output = entry
 534                break
 535        elif "data" in entry:
 536            data = entry.get("data")
 537            space = data.find(" ")
 538            if data[:space] == depotPath:
 539                output = entry
 540                break
 541    if output == None:
 542        return ""
 543    if output["code"] == "error":
 544        return ""
 545    clientPath = ""
 546    if "path" in output:
 547        clientPath = output.get("path")
 548    elif "data" in output:
 549        data = output.get("data")
 550        lastSpace = data.rfind(" ")
 551        clientPath = data[lastSpace + 1:]
 552
 553    if clientPath.endswith("..."):
 554        clientPath = clientPath[:-3]
 555    return clientPath
 556
 557def currentGitBranch():
 558    retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
 559    if retcode != 0:
 560        # on a detached head
 561        return None
 562    else:
 563        return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
 564
 565def isValidGitDir(path):
 566    if (os.path.exists(path + "/HEAD")
 567        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 568        return True;
 569    return False
 570
 571def parseRevision(ref):
 572    return read_pipe("git rev-parse %s" % ref).strip()
 573
 574def branchExists(ref):
 575    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 576                     ignore_error=True)
 577    return len(rev) > 0
 578
 579def extractLogMessageFromGitCommit(commit):
 580    logMessage = ""
 581
 582    ## fixme: title is first line of commit, not 1st paragraph.
 583    foundTitle = False
 584    for log in read_pipe_lines("git cat-file commit %s" % commit):
 585       if not foundTitle:
 586           if len(log) == 1:
 587               foundTitle = True
 588           continue
 589
 590       logMessage += log
 591    return logMessage
 592
 593def extractSettingsGitLog(log):
 594    values = {}
 595    for line in log.split("\n"):
 596        line = line.strip()
 597        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 598        if not m:
 599            continue
 600
 601        assignments = m.group(1).split (':')
 602        for a in assignments:
 603            vals = a.split ('=')
 604            key = vals[0].strip()
 605            val = ('='.join (vals[1:])).strip()
 606            if val.endswith ('\"') and val.startswith('"'):
 607                val = val[1:-1]
 608
 609            values[key] = val
 610
 611    paths = values.get("depot-paths")
 612    if not paths:
 613        paths = values.get("depot-path")
 614    if paths:
 615        values['depot-paths'] = paths.split(',')
 616    return values
 617
 618def gitBranchExists(branch):
 619    proc = subprocess.Popen(["git", "rev-parse", branch],
 620                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 621    return proc.wait() == 0;
 622
 623_gitConfig = {}
 624
 625def gitConfig(key, typeSpecifier=None):
 626    if not _gitConfig.has_key(key):
 627        cmd = [ "git", "config" ]
 628        if typeSpecifier:
 629            cmd += [ typeSpecifier ]
 630        cmd += [ key ]
 631        s = read_pipe(cmd, ignore_error=True)
 632        _gitConfig[key] = s.strip()
 633    return _gitConfig[key]
 634
 635def gitConfigBool(key):
 636    """Return a bool, using git config --bool.  It is True only if the
 637       variable is set to true, and False if set to false or not present
 638       in the config."""
 639
 640    if not _gitConfig.has_key(key):
 641        _gitConfig[key] = gitConfig(key, '--bool') == "true"
 642    return _gitConfig[key]
 643
 644def gitConfigInt(key):
 645    if not _gitConfig.has_key(key):
 646        cmd = [ "git", "config", "--int", key ]
 647        s = read_pipe(cmd, ignore_error=True)
 648        v = s.strip()
 649        try:
 650            _gitConfig[key] = int(gitConfig(key, '--int'))
 651        except ValueError:
 652            _gitConfig[key] = None
 653    return _gitConfig[key]
 654
 655def gitConfigList(key):
 656    if not _gitConfig.has_key(key):
 657        s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
 658        _gitConfig[key] = s.strip().split(os.linesep)
 659        if _gitConfig[key] == ['']:
 660            _gitConfig[key] = []
 661    return _gitConfig[key]
 662
 663def p4BranchesInGit(branchesAreInRemotes=True):
 664    """Find all the branches whose names start with "p4/", looking
 665       in remotes or heads as specified by the argument.  Return
 666       a dictionary of { branch: revision } for each one found.
 667       The branch names are the short names, without any
 668       "p4/" prefix."""
 669
 670    branches = {}
 671
 672    cmdline = "git rev-parse --symbolic "
 673    if branchesAreInRemotes:
 674        cmdline += "--remotes"
 675    else:
 676        cmdline += "--branches"
 677
 678    for line in read_pipe_lines(cmdline):
 679        line = line.strip()
 680
 681        # only import to p4/
 682        if not line.startswith('p4/'):
 683            continue
 684        # special symbolic ref to p4/master
 685        if line == "p4/HEAD":
 686            continue
 687
 688        # strip off p4/ prefix
 689        branch = line[len("p4/"):]
 690
 691        branches[branch] = parseRevision(line)
 692
 693    return branches
 694
 695def branch_exists(branch):
 696    """Make sure that the given ref name really exists."""
 697
 698    cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 699    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 700    out, _ = p.communicate()
 701    if p.returncode:
 702        return False
 703    # expect exactly one line of output: the branch name
 704    return out.rstrip() == branch
 705
 706def findUpstreamBranchPoint(head = "HEAD"):
 707    branches = p4BranchesInGit()
 708    # map from depot-path to branch name
 709    branchByDepotPath = {}
 710    for branch in branches.keys():
 711        tip = branches[branch]
 712        log = extractLogMessageFromGitCommit(tip)
 713        settings = extractSettingsGitLog(log)
 714        if settings.has_key("depot-paths"):
 715            paths = ",".join(settings["depot-paths"])
 716            branchByDepotPath[paths] = "remotes/p4/" + branch
 717
 718    settings = None
 719    parent = 0
 720    while parent < 65535:
 721        commit = head + "~%s" % parent
 722        log = extractLogMessageFromGitCommit(commit)
 723        settings = extractSettingsGitLog(log)
 724        if settings.has_key("depot-paths"):
 725            paths = ",".join(settings["depot-paths"])
 726            if branchByDepotPath.has_key(paths):
 727                return [branchByDepotPath[paths], settings]
 728
 729        parent = parent + 1
 730
 731    return ["", settings]
 732
 733def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 734    if not silent:
 735        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 736               % localRefPrefix)
 737
 738    originPrefix = "origin/p4/"
 739
 740    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 741        line = line.strip()
 742        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 743            continue
 744
 745        headName = line[len(originPrefix):]
 746        remoteHead = localRefPrefix + headName
 747        originHead = line
 748
 749        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 750        if (not original.has_key('depot-paths')
 751            or not original.has_key('change')):
 752            continue
 753
 754        update = False
 755        if not gitBranchExists(remoteHead):
 756            if verbose:
 757                print "creating %s" % remoteHead
 758            update = True
 759        else:
 760            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 761            if settings.has_key('change') > 0:
 762                if settings['depot-paths'] == original['depot-paths']:
 763                    originP4Change = int(original['change'])
 764                    p4Change = int(settings['change'])
 765                    if originP4Change > p4Change:
 766                        print ("%s (%s) is newer than %s (%s). "
 767                               "Updating p4 branch from origin."
 768                               % (originHead, originP4Change,
 769                                  remoteHead, p4Change))
 770                        update = True
 771                else:
 772                    print ("Ignoring: %s was imported from %s while "
 773                           "%s was imported from %s"
 774                           % (originHead, ','.join(original['depot-paths']),
 775                              remoteHead, ','.join(settings['depot-paths'])))
 776
 777        if update:
 778            system("git update-ref %s %s" % (remoteHead, originHead))
 779
 780def originP4BranchesExist():
 781        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 782
 783
 784def p4ParseNumericChangeRange(parts):
 785    changeStart = int(parts[0][1:])
 786    if parts[1] == '#head':
 787        changeEnd = p4_last_change()
 788    else:
 789        changeEnd = int(parts[1])
 790
 791    return (changeStart, changeEnd)
 792
 793def chooseBlockSize(blockSize):
 794    if blockSize:
 795        return blockSize
 796    else:
 797        return defaultBlockSize
 798
 799def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
 800    assert depotPaths
 801
 802    # Parse the change range into start and end. Try to find integer
 803    # revision ranges as these can be broken up into blocks to avoid
 804    # hitting server-side limits (maxrows, maxscanresults). But if
 805    # that doesn't work, fall back to using the raw revision specifier
 806    # strings, without using block mode.
 807
 808    if changeRange is None or changeRange == '':
 809        changeStart = 1
 810        changeEnd = p4_last_change()
 811        block_size = chooseBlockSize(requestedBlockSize)
 812    else:
 813        parts = changeRange.split(',')
 814        assert len(parts) == 2
 815        try:
 816            (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
 817            block_size = chooseBlockSize(requestedBlockSize)
 818        except:
 819            changeStart = parts[0][1:]
 820            changeEnd = parts[1]
 821            if requestedBlockSize:
 822                die("cannot use --changes-block-size with non-numeric revisions")
 823            block_size = None
 824
 825    # Accumulate change numbers in a dictionary to avoid duplicates
 826    changes = {}
 827
 828    for p in depotPaths:
 829        # Retrieve changes a block at a time, to prevent running
 830        # into a MaxResults/MaxScanRows error from the server.
 831
 832        while True:
 833            cmd = ['changes']
 834
 835            if block_size:
 836                end = min(changeEnd, changeStart + block_size)
 837                revisionRange = "%d,%d" % (changeStart, end)
 838            else:
 839                revisionRange = "%s,%s" % (changeStart, changeEnd)
 840
 841            cmd += ["%s...@%s" % (p, revisionRange)]
 842
 843            for line in p4_read_pipe_lines(cmd):
 844                changeNum = int(line.split(" ")[1])
 845                changes[changeNum] = True
 846
 847            if not block_size:
 848                break
 849
 850            if end >= changeEnd:
 851                break
 852
 853            changeStart = end + 1
 854
 855    changelist = changes.keys()
 856    changelist.sort()
 857    return changelist
 858
 859def p4PathStartsWith(path, prefix):
 860    # This method tries to remedy a potential mixed-case issue:
 861    #
 862    # If UserA adds  //depot/DirA/file1
 863    # and UserB adds //depot/dira/file2
 864    #
 865    # we may or may not have a problem. If you have core.ignorecase=true,
 866    # we treat DirA and dira as the same directory
 867    if gitConfigBool("core.ignorecase"):
 868        return path.lower().startswith(prefix.lower())
 869    return path.startswith(prefix)
 870
 871def getClientSpec():
 872    """Look at the p4 client spec, create a View() object that contains
 873       all the mappings, and return it."""
 874
 875    specList = p4CmdList("client -o")
 876    if len(specList) != 1:
 877        die('Output from "client -o" is %d lines, expecting 1' %
 878            len(specList))
 879
 880    # dictionary of all client parameters
 881    entry = specList[0]
 882
 883    # the //client/ name
 884    client_name = entry["Client"]
 885
 886    # just the keys that start with "View"
 887    view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 888
 889    # hold this new View
 890    view = View(client_name)
 891
 892    # append the lines, in order, to the view
 893    for view_num in range(len(view_keys)):
 894        k = "View%d" % view_num
 895        if k not in view_keys:
 896            die("Expected view key %s missing" % k)
 897        view.append(entry[k])
 898
 899    return view
 900
 901def getClientRoot():
 902    """Grab the client directory."""
 903
 904    output = p4CmdList("client -o")
 905    if len(output) != 1:
 906        die('Output from "client -o" is %d lines, expecting 1' % len(output))
 907
 908    entry = output[0]
 909    if "Root" not in entry:
 910        die('Client has no "Root"')
 911
 912    return entry["Root"]
 913
 914#
 915# P4 wildcards are not allowed in filenames.  P4 complains
 916# if you simply add them, but you can force it with "-f", in
 917# which case it translates them into %xx encoding internally.
 918#
 919def wildcard_decode(path):
 920    # Search for and fix just these four characters.  Do % last so
 921    # that fixing it does not inadvertently create new %-escapes.
 922    # Cannot have * in a filename in windows; untested as to
 923    # what p4 would do in such a case.
 924    if not platform.system() == "Windows":
 925        path = path.replace("%2A", "*")
 926    path = path.replace("%23", "#") \
 927               .replace("%40", "@") \
 928               .replace("%25", "%")
 929    return path
 930
 931def wildcard_encode(path):
 932    # do % first to avoid double-encoding the %s introduced here
 933    path = path.replace("%", "%25") \
 934               .replace("*", "%2A") \
 935               .replace("#", "%23") \
 936               .replace("@", "%40")
 937    return path
 938
 939def wildcard_present(path):
 940    m = re.search("[*#@%]", path)
 941    return m is not None
 942
 943class LargeFileSystem(object):
 944    """Base class for large file system support."""
 945
 946    def __init__(self, writeToGitStream):
 947        self.largeFiles = set()
 948        self.writeToGitStream = writeToGitStream
 949
 950    def generatePointer(self, cloneDestination, contentFile):
 951        """Return the content of a pointer file that is stored in Git instead of
 952           the actual content."""
 953        assert False, "Method 'generatePointer' required in " + self.__class__.__name__
 954
 955    def pushFile(self, localLargeFile):
 956        """Push the actual content which is not stored in the Git repository to
 957           a server."""
 958        assert False, "Method 'pushFile' required in " + self.__class__.__name__
 959
 960    def hasLargeFileExtension(self, relPath):
 961        return reduce(
 962            lambda a, b: a or b,
 963            [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
 964            False
 965        )
 966
 967    def generateTempFile(self, contents):
 968        contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
 969        for d in contents:
 970            contentFile.write(d)
 971        contentFile.close()
 972        return contentFile.name
 973
 974    def exceedsLargeFileThreshold(self, relPath, contents):
 975        if gitConfigInt('git-p4.largeFileThreshold'):
 976            contentsSize = sum(len(d) for d in contents)
 977            if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
 978                return True
 979        if gitConfigInt('git-p4.largeFileCompressedThreshold'):
 980            contentsSize = sum(len(d) for d in contents)
 981            if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
 982                return False
 983            contentTempFile = self.generateTempFile(contents)
 984            compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
 985            zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
 986            zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
 987            zf.close()
 988            compressedContentsSize = zf.infolist()[0].compress_size
 989            os.remove(contentTempFile)
 990            os.remove(compressedContentFile.name)
 991            if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
 992                return True
 993        return False
 994
 995    def addLargeFile(self, relPath):
 996        self.largeFiles.add(relPath)
 997
 998    def removeLargeFile(self, relPath):
 999        self.largeFiles.remove(relPath)
1000
1001    def isLargeFile(self, relPath):
1002        return relPath in self.largeFiles
1003
1004    def processContent(self, git_mode, relPath, contents):
1005        """Processes the content of git fast import. This method decides if a
1006           file is stored in the large file system and handles all necessary
1007           steps."""
1008        if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1009            contentTempFile = self.generateTempFile(contents)
1010            (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1011
1012            # Move temp file to final location in large file system
1013            largeFileDir = os.path.dirname(localLargeFile)
1014            if not os.path.isdir(largeFileDir):
1015                os.makedirs(largeFileDir)
1016            shutil.move(contentTempFile, localLargeFile)
1017            self.addLargeFile(relPath)
1018            if gitConfigBool('git-p4.largeFilePush'):
1019                self.pushFile(localLargeFile)
1020            if verbose:
1021                sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1022        return (git_mode, contents)
1023
1024class MockLFS(LargeFileSystem):
1025    """Mock large file system for testing."""
1026
1027    def generatePointer(self, contentFile):
1028        """The pointer content is the original content prefixed with "pointer-".
1029           The local filename of the large file storage is derived from the file content.
1030           """
1031        with open(contentFile, 'r') as f:
1032            content = next(f)
1033            gitMode = '100644'
1034            pointerContents = 'pointer-' + content
1035            localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1036            return (gitMode, pointerContents, localLargeFile)
1037
1038    def pushFile(self, localLargeFile):
1039        """The remote filename of the large file storage is the same as the local
1040           one but in a different directory.
1041           """
1042        remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1043        if not os.path.exists(remotePath):
1044            os.makedirs(remotePath)
1045        shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1046
1047class GitLFS(LargeFileSystem):
1048    """Git LFS as backend for the git-p4 large file system.
1049       See https://git-lfs.github.com/ for details."""
1050
1051    def __init__(self, *args):
1052        LargeFileSystem.__init__(self, *args)
1053        self.baseGitAttributes = []
1054
1055    def generatePointer(self, contentFile):
1056        """Generate a Git LFS pointer for the content. Return LFS Pointer file
1057           mode and content which is stored in the Git repository instead of
1058           the actual content. Return also the new location of the actual
1059           content.
1060           """
1061        pointerProcess = subprocess.Popen(
1062            ['git', 'lfs', 'pointer', '--file=' + contentFile],
1063            stdout=subprocess.PIPE
1064        )
1065        pointerFile = pointerProcess.stdout.read()
1066        if pointerProcess.wait():
1067            os.remove(contentFile)
1068            die('git-lfs pointer command failed. Did you install the extension?')
1069        pointerContents = [i+'\n' for i in pointerFile.split('\n')[2:][:-1]]
1070        oid = pointerContents[1].split(' ')[1].split(':')[1][:-1]
1071        localLargeFile = os.path.join(
1072            os.getcwd(),
1073            '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1074            oid,
1075        )
1076        # LFS Spec states that pointer files should not have the executable bit set.
1077        gitMode = '100644'
1078        return (gitMode, pointerContents, localLargeFile)
1079
1080    def pushFile(self, localLargeFile):
1081        uploadProcess = subprocess.Popen(
1082            ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1083        )
1084        if uploadProcess.wait():
1085            die('git-lfs push command failed. Did you define a remote?')
1086
1087    def generateGitAttributes(self):
1088        return (
1089            self.baseGitAttributes +
1090            [
1091                '\n',
1092                '#\n',
1093                '# Git LFS (see https://git-lfs.github.com/)\n',
1094                '#\n',
1095            ] +
1096            ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1097                for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1098            ] +
1099            ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1100                for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1101            ]
1102        )
1103
1104    def addLargeFile(self, relPath):
1105        LargeFileSystem.addLargeFile(self, relPath)
1106        self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1107
1108    def removeLargeFile(self, relPath):
1109        LargeFileSystem.removeLargeFile(self, relPath)
1110        self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1111
1112    def processContent(self, git_mode, relPath, contents):
1113        if relPath == '.gitattributes':
1114            self.baseGitAttributes = contents
1115            return (git_mode, self.generateGitAttributes())
1116        else:
1117            return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1118
1119class Command:
1120    def __init__(self):
1121        self.usage = "usage: %prog [options]"
1122        self.needsGit = True
1123        self.verbose = False
1124
1125class P4UserMap:
1126    def __init__(self):
1127        self.userMapFromPerforceServer = False
1128        self.myP4UserId = None
1129
1130    def p4UserId(self):
1131        if self.myP4UserId:
1132            return self.myP4UserId
1133
1134        results = p4CmdList("user -o")
1135        for r in results:
1136            if r.has_key('User'):
1137                self.myP4UserId = r['User']
1138                return r['User']
1139        die("Could not find your p4 user id")
1140
1141    def p4UserIsMe(self, p4User):
1142        # return True if the given p4 user is actually me
1143        me = self.p4UserId()
1144        if not p4User or p4User != me:
1145            return False
1146        else:
1147            return True
1148
1149    def getUserCacheFilename(self):
1150        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1151        return home + "/.gitp4-usercache.txt"
1152
1153    def getUserMapFromPerforceServer(self):
1154        if self.userMapFromPerforceServer:
1155            return
1156        self.users = {}
1157        self.emails = {}
1158
1159        for output in p4CmdList("users"):
1160            if not output.has_key("User"):
1161                continue
1162            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1163            self.emails[output["Email"]] = output["User"]
1164
1165
1166        s = ''
1167        for (key, val) in self.users.items():
1168            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1169
1170        open(self.getUserCacheFilename(), "wb").write(s)
1171        self.userMapFromPerforceServer = True
1172
1173    def loadUserMapFromCache(self):
1174        self.users = {}
1175        self.userMapFromPerforceServer = False
1176        try:
1177            cache = open(self.getUserCacheFilename(), "rb")
1178            lines = cache.readlines()
1179            cache.close()
1180            for line in lines:
1181                entry = line.strip().split("\t")
1182                self.users[entry[0]] = entry[1]
1183        except IOError:
1184            self.getUserMapFromPerforceServer()
1185
1186class P4Debug(Command):
1187    def __init__(self):
1188        Command.__init__(self)
1189        self.options = []
1190        self.description = "A tool to debug the output of p4 -G."
1191        self.needsGit = False
1192
1193    def run(self, args):
1194        j = 0
1195        for output in p4CmdList(args):
1196            print 'Element: %d' % j
1197            j += 1
1198            print output
1199        return True
1200
1201class P4RollBack(Command):
1202    def __init__(self):
1203        Command.__init__(self)
1204        self.options = [
1205            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1206        ]
1207        self.description = "A tool to debug the multi-branch import. Don't use :)"
1208        self.rollbackLocalBranches = False
1209
1210    def run(self, args):
1211        if len(args) != 1:
1212            return False
1213        maxChange = int(args[0])
1214
1215        if "p4ExitCode" in p4Cmd("changes -m 1"):
1216            die("Problems executing p4");
1217
1218        if self.rollbackLocalBranches:
1219            refPrefix = "refs/heads/"
1220            lines = read_pipe_lines("git rev-parse --symbolic --branches")
1221        else:
1222            refPrefix = "refs/remotes/"
1223            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1224
1225        for line in lines:
1226            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1227                line = line.strip()
1228                ref = refPrefix + line
1229                log = extractLogMessageFromGitCommit(ref)
1230                settings = extractSettingsGitLog(log)
1231
1232                depotPaths = settings['depot-paths']
1233                change = settings['change']
1234
1235                changed = False
1236
1237                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1238                                                           for p in depotPaths]))) == 0:
1239                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1240                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1241                    continue
1242
1243                while change and int(change) > maxChange:
1244                    changed = True
1245                    if self.verbose:
1246                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1247                    system("git update-ref %s \"%s^\"" % (ref, ref))
1248                    log = extractLogMessageFromGitCommit(ref)
1249                    settings =  extractSettingsGitLog(log)
1250
1251
1252                    depotPaths = settings['depot-paths']
1253                    change = settings['change']
1254
1255                if changed:
1256                    print "%s rewound to %s" % (ref, change)
1257
1258        return True
1259
1260class P4Submit(Command, P4UserMap):
1261
1262    conflict_behavior_choices = ("ask", "skip", "quit")
1263
1264    def __init__(self):
1265        Command.__init__(self)
1266        P4UserMap.__init__(self)
1267        self.options = [
1268                optparse.make_option("--origin", dest="origin"),
1269                optparse.make_option("-M", dest="detectRenames", action="store_true"),
1270                # preserve the user, requires relevant p4 permissions
1271                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1272                optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1273                optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1274                optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1275                optparse.make_option("--conflict", dest="conflict_behavior",
1276                                     choices=self.conflict_behavior_choices),
1277                optparse.make_option("--branch", dest="branch"),
1278        ]
1279        self.description = "Submit changes from git to the perforce depot."
1280        self.usage += " [name of git branch to submit into perforce depot]"
1281        self.origin = ""
1282        self.detectRenames = False
1283        self.preserveUser = gitConfigBool("git-p4.preserveUser")
1284        self.dry_run = False
1285        self.prepare_p4_only = False
1286        self.conflict_behavior = None
1287        self.isWindows = (platform.system() == "Windows")
1288        self.exportLabels = False
1289        self.p4HasMoveCommand = p4_has_move_command()
1290        self.branch = None
1291
1292        if gitConfig('git-p4.largeFileSystem'):
1293            die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1294
1295    def check(self):
1296        if len(p4CmdList("opened ...")) > 0:
1297            die("You have files opened with perforce! Close them before starting the sync.")
1298
1299    def separate_jobs_from_description(self, message):
1300        """Extract and return a possible Jobs field in the commit
1301           message.  It goes into a separate section in the p4 change
1302           specification.
1303
1304           A jobs line starts with "Jobs:" and looks like a new field
1305           in a form.  Values are white-space separated on the same
1306           line or on following lines that start with a tab.
1307
1308           This does not parse and extract the full git commit message
1309           like a p4 form.  It just sees the Jobs: line as a marker
1310           to pass everything from then on directly into the p4 form,
1311           but outside the description section.
1312
1313           Return a tuple (stripped log message, jobs string)."""
1314
1315        m = re.search(r'^Jobs:', message, re.MULTILINE)
1316        if m is None:
1317            return (message, None)
1318
1319        jobtext = message[m.start():]
1320        stripped_message = message[:m.start()].rstrip()
1321        return (stripped_message, jobtext)
1322
1323    def prepareLogMessage(self, template, message, jobs):
1324        """Edits the template returned from "p4 change -o" to insert
1325           the message in the Description field, and the jobs text in
1326           the Jobs field."""
1327        result = ""
1328
1329        inDescriptionSection = False
1330
1331        for line in template.split("\n"):
1332            if line.startswith("#"):
1333                result += line + "\n"
1334                continue
1335
1336            if inDescriptionSection:
1337                if line.startswith("Files:") or line.startswith("Jobs:"):
1338                    inDescriptionSection = False
1339                    # insert Jobs section
1340                    if jobs:
1341                        result += jobs + "\n"
1342                else:
1343                    continue
1344            else:
1345                if line.startswith("Description:"):
1346                    inDescriptionSection = True
1347                    line += "\n"
1348                    for messageLine in message.split("\n"):
1349                        line += "\t" + messageLine + "\n"
1350
1351            result += line + "\n"
1352
1353        return result
1354
1355    def patchRCSKeywords(self, file, pattern):
1356        # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1357        (handle, outFileName) = tempfile.mkstemp(dir='.')
1358        try:
1359            outFile = os.fdopen(handle, "w+")
1360            inFile = open(file, "r")
1361            regexp = re.compile(pattern, re.VERBOSE)
1362            for line in inFile.readlines():
1363                line = regexp.sub(r'$\1$', line)
1364                outFile.write(line)
1365            inFile.close()
1366            outFile.close()
1367            # Forcibly overwrite the original file
1368            os.unlink(file)
1369            shutil.move(outFileName, file)
1370        except:
1371            # cleanup our temporary file
1372            os.unlink(outFileName)
1373            print "Failed to strip RCS keywords in %s" % file
1374            raise
1375
1376        print "Patched up RCS keywords in %s" % file
1377
1378    def p4UserForCommit(self,id):
1379        # Return the tuple (perforce user,git email) for a given git commit id
1380        self.getUserMapFromPerforceServer()
1381        gitEmail = read_pipe(["git", "log", "--max-count=1",
1382                              "--format=%ae", id])
1383        gitEmail = gitEmail.strip()
1384        if not self.emails.has_key(gitEmail):
1385            return (None,gitEmail)
1386        else:
1387            return (self.emails[gitEmail],gitEmail)
1388
1389    def checkValidP4Users(self,commits):
1390        # check if any git authors cannot be mapped to p4 users
1391        for id in commits:
1392            (user,email) = self.p4UserForCommit(id)
1393            if not user:
1394                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1395                if gitConfigBool("git-p4.allowMissingP4Users"):
1396                    print "%s" % msg
1397                else:
1398                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1399
1400    def lastP4Changelist(self):
1401        # Get back the last changelist number submitted in this client spec. This
1402        # then gets used to patch up the username in the change. If the same
1403        # client spec is being used by multiple processes then this might go
1404        # wrong.
1405        results = p4CmdList("client -o")        # find the current client
1406        client = None
1407        for r in results:
1408            if r.has_key('Client'):
1409                client = r['Client']
1410                break
1411        if not client:
1412            die("could not get client spec")
1413        results = p4CmdList(["changes", "-c", client, "-m", "1"])
1414        for r in results:
1415            if r.has_key('change'):
1416                return r['change']
1417        die("Could not get changelist number for last submit - cannot patch up user details")
1418
1419    def modifyChangelistUser(self, changelist, newUser):
1420        # fixup the user field of a changelist after it has been submitted.
1421        changes = p4CmdList("change -o %s" % changelist)
1422        if len(changes) != 1:
1423            die("Bad output from p4 change modifying %s to user %s" %
1424                (changelist, newUser))
1425
1426        c = changes[0]
1427        if c['User'] == newUser: return   # nothing to do
1428        c['User'] = newUser
1429        input = marshal.dumps(c)
1430
1431        result = p4CmdList("change -f -i", stdin=input)
1432        for r in result:
1433            if r.has_key('code'):
1434                if r['code'] == 'error':
1435                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1436            if r.has_key('data'):
1437                print("Updated user field for changelist %s to %s" % (changelist, newUser))
1438                return
1439        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1440
1441    def canChangeChangelists(self):
1442        # check to see if we have p4 admin or super-user permissions, either of
1443        # which are required to modify changelists.
1444        results = p4CmdList(["protects", self.depotPath])
1445        for r in results:
1446            if r.has_key('perm'):
1447                if r['perm'] == 'admin':
1448                    return 1
1449                if r['perm'] == 'super':
1450                    return 1
1451        return 0
1452
1453    def prepareSubmitTemplate(self):
1454        """Run "p4 change -o" to grab a change specification template.
1455           This does not use "p4 -G", as it is nice to keep the submission
1456           template in original order, since a human might edit it.
1457
1458           Remove lines in the Files section that show changes to files
1459           outside the depot path we're committing into."""
1460
1461        template = ""
1462        inFilesSection = False
1463        for line in p4_read_pipe_lines(['change', '-o']):
1464            if line.endswith("\r\n"):
1465                line = line[:-2] + "\n"
1466            if inFilesSection:
1467                if line.startswith("\t"):
1468                    # path starts and ends with a tab
1469                    path = line[1:]
1470                    lastTab = path.rfind("\t")
1471                    if lastTab != -1:
1472                        path = path[:lastTab]
1473                        if not p4PathStartsWith(path, self.depotPath):
1474                            continue
1475                else:
1476                    inFilesSection = False
1477            else:
1478                if line.startswith("Files:"):
1479                    inFilesSection = True
1480
1481            template += line
1482
1483        return template
1484
1485    def edit_template(self, template_file):
1486        """Invoke the editor to let the user change the submission
1487           message.  Return true if okay to continue with the submit."""
1488
1489        # if configured to skip the editing part, just submit
1490        if gitConfigBool("git-p4.skipSubmitEdit"):
1491            return True
1492
1493        # look at the modification time, to check later if the user saved
1494        # the file
1495        mtime = os.stat(template_file).st_mtime
1496
1497        # invoke the editor
1498        if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1499            editor = os.environ.get("P4EDITOR")
1500        else:
1501            editor = read_pipe("git var GIT_EDITOR").strip()
1502        system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1503
1504        # If the file was not saved, prompt to see if this patch should
1505        # be skipped.  But skip this verification step if configured so.
1506        if gitConfigBool("git-p4.skipSubmitEditCheck"):
1507            return True
1508
1509        # modification time updated means user saved the file
1510        if os.stat(template_file).st_mtime > mtime:
1511            return True
1512
1513        while True:
1514            response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1515            if response == 'y':
1516                return True
1517            if response == 'n':
1518                return False
1519
1520    def get_diff_description(self, editedFiles, filesToAdd):
1521        # diff
1522        if os.environ.has_key("P4DIFF"):
1523            del(os.environ["P4DIFF"])
1524        diff = ""
1525        for editedFile in editedFiles:
1526            diff += p4_read_pipe(['diff', '-du',
1527                                  wildcard_encode(editedFile)])
1528
1529        # new file diff
1530        newdiff = ""
1531        for newFile in filesToAdd:
1532            newdiff += "==== new file ====\n"
1533            newdiff += "--- /dev/null\n"
1534            newdiff += "+++ %s\n" % newFile
1535            f = open(newFile, "r")
1536            for line in f.readlines():
1537                newdiff += "+" + line
1538            f.close()
1539
1540        return (diff + newdiff).replace('\r\n', '\n')
1541
1542    def applyCommit(self, id):
1543        """Apply one commit, return True if it succeeded."""
1544
1545        print "Applying", read_pipe(["git", "show", "-s",
1546                                     "--format=format:%h %s", id])
1547
1548        (p4User, gitEmail) = self.p4UserForCommit(id)
1549
1550        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1551        filesToAdd = set()
1552        filesToDelete = set()
1553        editedFiles = set()
1554        pureRenameCopy = set()
1555        filesToChangeExecBit = {}
1556
1557        for line in diff:
1558            diff = parseDiffTreeEntry(line)
1559            modifier = diff['status']
1560            path = diff['src']
1561            if modifier == "M":
1562                p4_edit(path)
1563                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1564                    filesToChangeExecBit[path] = diff['dst_mode']
1565                editedFiles.add(path)
1566            elif modifier == "A":
1567                filesToAdd.add(path)
1568                filesToChangeExecBit[path] = diff['dst_mode']
1569                if path in filesToDelete:
1570                    filesToDelete.remove(path)
1571            elif modifier == "D":
1572                filesToDelete.add(path)
1573                if path in filesToAdd:
1574                    filesToAdd.remove(path)
1575            elif modifier == "C":
1576                src, dest = diff['src'], diff['dst']
1577                p4_integrate(src, dest)
1578                pureRenameCopy.add(dest)
1579                if diff['src_sha1'] != diff['dst_sha1']:
1580                    p4_edit(dest)
1581                    pureRenameCopy.discard(dest)
1582                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1583                    p4_edit(dest)
1584                    pureRenameCopy.discard(dest)
1585                    filesToChangeExecBit[dest] = diff['dst_mode']
1586                if self.isWindows:
1587                    # turn off read-only attribute
1588                    os.chmod(dest, stat.S_IWRITE)
1589                os.unlink(dest)
1590                editedFiles.add(dest)
1591            elif modifier == "R":
1592                src, dest = diff['src'], diff['dst']
1593                if self.p4HasMoveCommand:
1594                    p4_edit(src)        # src must be open before move
1595                    p4_move(src, dest)  # opens for (move/delete, move/add)
1596                else:
1597                    p4_integrate(src, dest)
1598                    if diff['src_sha1'] != diff['dst_sha1']:
1599                        p4_edit(dest)
1600                    else:
1601                        pureRenameCopy.add(dest)
1602                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1603                    if not self.p4HasMoveCommand:
1604                        p4_edit(dest)   # with move: already open, writable
1605                    filesToChangeExecBit[dest] = diff['dst_mode']
1606                if not self.p4HasMoveCommand:
1607                    if self.isWindows:
1608                        os.chmod(dest, stat.S_IWRITE)
1609                    os.unlink(dest)
1610                    filesToDelete.add(src)
1611                editedFiles.add(dest)
1612            else:
1613                die("unknown modifier %s for %s" % (modifier, path))
1614
1615        diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1616        patchcmd = diffcmd + " | git apply "
1617        tryPatchCmd = patchcmd + "--check -"
1618        applyPatchCmd = patchcmd + "--check --apply -"
1619        patch_succeeded = True
1620
1621        if os.system(tryPatchCmd) != 0:
1622            fixed_rcs_keywords = False
1623            patch_succeeded = False
1624            print "Unfortunately applying the change failed!"
1625
1626            # Patch failed, maybe it's just RCS keyword woes. Look through
1627            # the patch to see if that's possible.
1628            if gitConfigBool("git-p4.attemptRCSCleanup"):
1629                file = None
1630                pattern = None
1631                kwfiles = {}
1632                for file in editedFiles | filesToDelete:
1633                    # did this file's delta contain RCS keywords?
1634                    pattern = p4_keywords_regexp_for_file(file)
1635
1636                    if pattern:
1637                        # this file is a possibility...look for RCS keywords.
1638                        regexp = re.compile(pattern, re.VERBOSE)
1639                        for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1640                            if regexp.search(line):
1641                                if verbose:
1642                                    print "got keyword match on %s in %s in %s" % (pattern, line, file)
1643                                kwfiles[file] = pattern
1644                                break
1645
1646                for file in kwfiles:
1647                    if verbose:
1648                        print "zapping %s with %s" % (line,pattern)
1649                    # File is being deleted, so not open in p4.  Must
1650                    # disable the read-only bit on windows.
1651                    if self.isWindows and file not in editedFiles:
1652                        os.chmod(file, stat.S_IWRITE)
1653                    self.patchRCSKeywords(file, kwfiles[file])
1654                    fixed_rcs_keywords = True
1655
1656            if fixed_rcs_keywords:
1657                print "Retrying the patch with RCS keywords cleaned up"
1658                if os.system(tryPatchCmd) == 0:
1659                    patch_succeeded = True
1660
1661        if not patch_succeeded:
1662            for f in editedFiles:
1663                p4_revert(f)
1664            return False
1665
1666        #
1667        # Apply the patch for real, and do add/delete/+x handling.
1668        #
1669        system(applyPatchCmd)
1670
1671        for f in filesToAdd:
1672            p4_add(f)
1673        for f in filesToDelete:
1674            p4_revert(f)
1675            p4_delete(f)
1676
1677        # Set/clear executable bits
1678        for f in filesToChangeExecBit.keys():
1679            mode = filesToChangeExecBit[f]
1680            setP4ExecBit(f, mode)
1681
1682        #
1683        # Build p4 change description, starting with the contents
1684        # of the git commit message.
1685        #
1686        logMessage = extractLogMessageFromGitCommit(id)
1687        logMessage = logMessage.strip()
1688        (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1689
1690        template = self.prepareSubmitTemplate()
1691        submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1692
1693        if self.preserveUser:
1694           submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1695
1696        if self.checkAuthorship and not self.p4UserIsMe(p4User):
1697            submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1698            submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1699            submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1700
1701        separatorLine = "######## everything below this line is just the diff #######\n"
1702        if not self.prepare_p4_only:
1703            submitTemplate += separatorLine
1704            submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1705
1706        (handle, fileName) = tempfile.mkstemp()
1707        tmpFile = os.fdopen(handle, "w+b")
1708        if self.isWindows:
1709            submitTemplate = submitTemplate.replace("\n", "\r\n")
1710        tmpFile.write(submitTemplate)
1711        tmpFile.close()
1712
1713        if self.prepare_p4_only:
1714            #
1715            # Leave the p4 tree prepared, and the submit template around
1716            # and let the user decide what to do next
1717            #
1718            print
1719            print "P4 workspace prepared for submission."
1720            print "To submit or revert, go to client workspace"
1721            print "  " + self.clientPath
1722            print
1723            print "To submit, use \"p4 submit\" to write a new description,"
1724            print "or \"p4 submit -i <%s\" to use the one prepared by" \
1725                  " \"git p4\"." % fileName
1726            print "You can delete the file \"%s\" when finished." % fileName
1727
1728            if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1729                print "To preserve change ownership by user %s, you must\n" \
1730                      "do \"p4 change -f <change>\" after submitting and\n" \
1731                      "edit the User field."
1732            if pureRenameCopy:
1733                print "After submitting, renamed files must be re-synced."
1734                print "Invoke \"p4 sync -f\" on each of these files:"
1735                for f in pureRenameCopy:
1736                    print "  " + f
1737
1738            print
1739            print "To revert the changes, use \"p4 revert ...\", and delete"
1740            print "the submit template file \"%s\"" % fileName
1741            if filesToAdd:
1742                print "Since the commit adds new files, they must be deleted:"
1743                for f in filesToAdd:
1744                    print "  " + f
1745            print
1746            return True
1747
1748        #
1749        # Let the user edit the change description, then submit it.
1750        #
1751        submitted = False
1752
1753        try:
1754            if self.edit_template(fileName):
1755                # read the edited message and submit
1756                tmpFile = open(fileName, "rb")
1757                message = tmpFile.read()
1758                tmpFile.close()
1759                if self.isWindows:
1760                    message = message.replace("\r\n", "\n")
1761                submitTemplate = message[:message.index(separatorLine)]
1762                p4_write_pipe(['submit', '-i'], submitTemplate)
1763
1764                if self.preserveUser:
1765                    if p4User:
1766                        # Get last changelist number. Cannot easily get it from
1767                        # the submit command output as the output is
1768                        # unmarshalled.
1769                        changelist = self.lastP4Changelist()
1770                        self.modifyChangelistUser(changelist, p4User)
1771
1772                # The rename/copy happened by applying a patch that created a
1773                # new file.  This leaves it writable, which confuses p4.
1774                for f in pureRenameCopy:
1775                    p4_sync(f, "-f")
1776                submitted = True
1777
1778        finally:
1779            # skip this patch
1780            if not submitted:
1781                print "Submission cancelled, undoing p4 changes."
1782                for f in editedFiles:
1783                    p4_revert(f)
1784                for f in filesToAdd:
1785                    p4_revert(f)
1786                    os.remove(f)
1787                for f in filesToDelete:
1788                    p4_revert(f)
1789
1790        os.remove(fileName)
1791        return submitted
1792
1793    # Export git tags as p4 labels. Create a p4 label and then tag
1794    # with that.
1795    def exportGitTags(self, gitTags):
1796        validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1797        if len(validLabelRegexp) == 0:
1798            validLabelRegexp = defaultLabelRegexp
1799        m = re.compile(validLabelRegexp)
1800
1801        for name in gitTags:
1802
1803            if not m.match(name):
1804                if verbose:
1805                    print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1806                continue
1807
1808            # Get the p4 commit this corresponds to
1809            logMessage = extractLogMessageFromGitCommit(name)
1810            values = extractSettingsGitLog(logMessage)
1811
1812            if not values.has_key('change'):
1813                # a tag pointing to something not sent to p4; ignore
1814                if verbose:
1815                    print "git tag %s does not give a p4 commit" % name
1816                continue
1817            else:
1818                changelist = values['change']
1819
1820            # Get the tag details.
1821            inHeader = True
1822            isAnnotated = False
1823            body = []
1824            for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1825                l = l.strip()
1826                if inHeader:
1827                    if re.match(r'tag\s+', l):
1828                        isAnnotated = True
1829                    elif re.match(r'\s*$', l):
1830                        inHeader = False
1831                        continue
1832                else:
1833                    body.append(l)
1834
1835            if not isAnnotated:
1836                body = ["lightweight tag imported by git p4\n"]
1837
1838            # Create the label - use the same view as the client spec we are using
1839            clientSpec = getClientSpec()
1840
1841            labelTemplate  = "Label: %s\n" % name
1842            labelTemplate += "Description:\n"
1843            for b in body:
1844                labelTemplate += "\t" + b + "\n"
1845            labelTemplate += "View:\n"
1846            for depot_side in clientSpec.mappings:
1847                labelTemplate += "\t%s\n" % depot_side
1848
1849            if self.dry_run:
1850                print "Would create p4 label %s for tag" % name
1851            elif self.prepare_p4_only:
1852                print "Not creating p4 label %s for tag due to option" \
1853                      " --prepare-p4-only" % name
1854            else:
1855                p4_write_pipe(["label", "-i"], labelTemplate)
1856
1857                # Use the label
1858                p4_system(["tag", "-l", name] +
1859                          ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1860
1861                if verbose:
1862                    print "created p4 label for tag %s" % name
1863
1864    def run(self, args):
1865        if len(args) == 0:
1866            self.master = currentGitBranch()
1867        elif len(args) == 1:
1868            self.master = args[0]
1869            if not branchExists(self.master):
1870                die("Branch %s does not exist" % self.master)
1871        else:
1872            return False
1873
1874        if self.master:
1875            allowSubmit = gitConfig("git-p4.allowSubmit")
1876            if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1877                die("%s is not in git-p4.allowSubmit" % self.master)
1878
1879        [upstream, settings] = findUpstreamBranchPoint()
1880        self.depotPath = settings['depot-paths'][0]
1881        if len(self.origin) == 0:
1882            self.origin = upstream
1883
1884        if self.preserveUser:
1885            if not self.canChangeChangelists():
1886                die("Cannot preserve user names without p4 super-user or admin permissions")
1887
1888        # if not set from the command line, try the config file
1889        if self.conflict_behavior is None:
1890            val = gitConfig("git-p4.conflict")
1891            if val:
1892                if val not in self.conflict_behavior_choices:
1893                    die("Invalid value '%s' for config git-p4.conflict" % val)
1894            else:
1895                val = "ask"
1896            self.conflict_behavior = val
1897
1898        if self.verbose:
1899            print "Origin branch is " + self.origin
1900
1901        if len(self.depotPath) == 0:
1902            print "Internal error: cannot locate perforce depot path from existing branches"
1903            sys.exit(128)
1904
1905        self.useClientSpec = False
1906        if gitConfigBool("git-p4.useclientspec"):
1907            self.useClientSpec = True
1908        if self.useClientSpec:
1909            self.clientSpecDirs = getClientSpec()
1910
1911        # Check for the existance of P4 branches
1912        branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1913
1914        if self.useClientSpec and not branchesDetected:
1915            # all files are relative to the client spec
1916            self.clientPath = getClientRoot()
1917        else:
1918            self.clientPath = p4Where(self.depotPath)
1919
1920        if self.clientPath == "":
1921            die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1922
1923        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1924        self.oldWorkingDirectory = os.getcwd()
1925
1926        # ensure the clientPath exists
1927        new_client_dir = False
1928        if not os.path.exists(self.clientPath):
1929            new_client_dir = True
1930            os.makedirs(self.clientPath)
1931
1932        chdir(self.clientPath, is_client_path=True)
1933        if self.dry_run:
1934            print "Would synchronize p4 checkout in %s" % self.clientPath
1935        else:
1936            print "Synchronizing p4 checkout..."
1937            if new_client_dir:
1938                # old one was destroyed, and maybe nobody told p4
1939                p4_sync("...", "-f")
1940            else:
1941                p4_sync("...")
1942        self.check()
1943
1944        commits = []
1945        if self.master:
1946            commitish = self.master
1947        else:
1948            commitish = 'HEAD'
1949
1950        for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1951            commits.append(line.strip())
1952        commits.reverse()
1953
1954        if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1955            self.checkAuthorship = False
1956        else:
1957            self.checkAuthorship = True
1958
1959        if self.preserveUser:
1960            self.checkValidP4Users(commits)
1961
1962        #
1963        # Build up a set of options to be passed to diff when
1964        # submitting each commit to p4.
1965        #
1966        if self.detectRenames:
1967            # command-line -M arg
1968            self.diffOpts = "-M"
1969        else:
1970            # If not explicitly set check the config variable
1971            detectRenames = gitConfig("git-p4.detectRenames")
1972
1973            if detectRenames.lower() == "false" or detectRenames == "":
1974                self.diffOpts = ""
1975            elif detectRenames.lower() == "true":
1976                self.diffOpts = "-M"
1977            else:
1978                self.diffOpts = "-M%s" % detectRenames
1979
1980        # no command-line arg for -C or --find-copies-harder, just
1981        # config variables
1982        detectCopies = gitConfig("git-p4.detectCopies")
1983        if detectCopies.lower() == "false" or detectCopies == "":
1984            pass
1985        elif detectCopies.lower() == "true":
1986            self.diffOpts += " -C"
1987        else:
1988            self.diffOpts += " -C%s" % detectCopies
1989
1990        if gitConfigBool("git-p4.detectCopiesHarder"):
1991            self.diffOpts += " --find-copies-harder"
1992
1993        #
1994        # Apply the commits, one at a time.  On failure, ask if should
1995        # continue to try the rest of the patches, or quit.
1996        #
1997        if self.dry_run:
1998            print "Would apply"
1999        applied = []
2000        last = len(commits) - 1
2001        for i, commit in enumerate(commits):
2002            if self.dry_run:
2003                print " ", read_pipe(["git", "show", "-s",
2004                                      "--format=format:%h %s", commit])
2005                ok = True
2006            else:
2007                ok = self.applyCommit(commit)
2008            if ok:
2009                applied.append(commit)
2010            else:
2011                if self.prepare_p4_only and i < last:
2012                    print "Processing only the first commit due to option" \
2013                          " --prepare-p4-only"
2014                    break
2015                if i < last:
2016                    quit = False
2017                    while True:
2018                        # prompt for what to do, or use the option/variable
2019                        if self.conflict_behavior == "ask":
2020                            print "What do you want to do?"
2021                            response = raw_input("[s]kip this commit but apply"
2022                                                 " the rest, or [q]uit? ")
2023                            if not response:
2024                                continue
2025                        elif self.conflict_behavior == "skip":
2026                            response = "s"
2027                        elif self.conflict_behavior == "quit":
2028                            response = "q"
2029                        else:
2030                            die("Unknown conflict_behavior '%s'" %
2031                                self.conflict_behavior)
2032
2033                        if response[0] == "s":
2034                            print "Skipping this commit, but applying the rest"
2035                            break
2036                        if response[0] == "q":
2037                            print "Quitting"
2038                            quit = True
2039                            break
2040                    if quit:
2041                        break
2042
2043        chdir(self.oldWorkingDirectory)
2044
2045        if self.dry_run:
2046            pass
2047        elif self.prepare_p4_only:
2048            pass
2049        elif len(commits) == len(applied):
2050            print "All commits applied!"
2051
2052            sync = P4Sync()
2053            if self.branch:
2054                sync.branch = self.branch
2055            sync.run([])
2056
2057            rebase = P4Rebase()
2058            rebase.rebase()
2059
2060        else:
2061            if len(applied) == 0:
2062                print "No commits applied."
2063            else:
2064                print "Applied only the commits marked with '*':"
2065                for c in commits:
2066                    if c in applied:
2067                        star = "*"
2068                    else:
2069                        star = " "
2070                    print star, read_pipe(["git", "show", "-s",
2071                                           "--format=format:%h %s",  c])
2072                print "You will have to do 'git p4 sync' and rebase."
2073
2074        if gitConfigBool("git-p4.exportLabels"):
2075            self.exportLabels = True
2076
2077        if self.exportLabels:
2078            p4Labels = getP4Labels(self.depotPath)
2079            gitTags = getGitTags()
2080
2081            missingGitTags = gitTags - p4Labels
2082            self.exportGitTags(missingGitTags)
2083
2084        # exit with error unless everything applied perfectly
2085        if len(commits) != len(applied):
2086                sys.exit(1)
2087
2088        return True
2089
2090class View(object):
2091    """Represent a p4 view ("p4 help views"), and map files in a
2092       repo according to the view."""
2093
2094    def __init__(self, client_name):
2095        self.mappings = []
2096        self.client_prefix = "//%s/" % client_name
2097        # cache results of "p4 where" to lookup client file locations
2098        self.client_spec_path_cache = {}
2099
2100    def append(self, view_line):
2101        """Parse a view line, splitting it into depot and client
2102           sides.  Append to self.mappings, preserving order.  This
2103           is only needed for tag creation."""
2104
2105        # Split the view line into exactly two words.  P4 enforces
2106        # structure on these lines that simplifies this quite a bit.
2107        #
2108        # Either or both words may be double-quoted.
2109        # Single quotes do not matter.
2110        # Double-quote marks cannot occur inside the words.
2111        # A + or - prefix is also inside the quotes.
2112        # There are no quotes unless they contain a space.
2113        # The line is already white-space stripped.
2114        # The two words are separated by a single space.
2115        #
2116        if view_line[0] == '"':
2117            # First word is double quoted.  Find its end.
2118            close_quote_index = view_line.find('"', 1)
2119            if close_quote_index <= 0:
2120                die("No first-word closing quote found: %s" % view_line)
2121            depot_side = view_line[1:close_quote_index]
2122            # skip closing quote and space
2123            rhs_index = close_quote_index + 1 + 1
2124        else:
2125            space_index = view_line.find(" ")
2126            if space_index <= 0:
2127                die("No word-splitting space found: %s" % view_line)
2128            depot_side = view_line[0:space_index]
2129            rhs_index = space_index + 1
2130
2131        # prefix + means overlay on previous mapping
2132        if depot_side.startswith("+"):
2133            depot_side = depot_side[1:]
2134
2135        # prefix - means exclude this path, leave out of mappings
2136        exclude = False
2137        if depot_side.startswith("-"):
2138            exclude = True
2139            depot_side = depot_side[1:]
2140
2141        if not exclude:
2142            self.mappings.append(depot_side)
2143
2144    def convert_client_path(self, clientFile):
2145        # chop off //client/ part to make it relative
2146        if not clientFile.startswith(self.client_prefix):
2147            die("No prefix '%s' on clientFile '%s'" %
2148                (self.client_prefix, clientFile))
2149        return clientFile[len(self.client_prefix):]
2150
2151    def update_client_spec_path_cache(self, files):
2152        """ Caching file paths by "p4 where" batch query """
2153
2154        # List depot file paths exclude that already cached
2155        fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2156
2157        if len(fileArgs) == 0:
2158            return  # All files in cache
2159
2160        where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2161        for res in where_result:
2162            if "code" in res and res["code"] == "error":
2163                # assume error is "... file(s) not in client view"
2164                continue
2165            if "clientFile" not in res:
2166                die("No clientFile in 'p4 where' output")
2167            if "unmap" in res:
2168                # it will list all of them, but only one not unmap-ped
2169                continue
2170            if gitConfigBool("core.ignorecase"):
2171                res['depotFile'] = res['depotFile'].lower()
2172            self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2173
2174        # not found files or unmap files set to ""
2175        for depotFile in fileArgs:
2176            if gitConfigBool("core.ignorecase"):
2177                depotFile = depotFile.lower()
2178            if depotFile not in self.client_spec_path_cache:
2179                self.client_spec_path_cache[depotFile] = ""
2180
2181    def map_in_client(self, depot_path):
2182        """Return the relative location in the client where this
2183           depot file should live.  Returns "" if the file should
2184           not be mapped in the client."""
2185
2186        if gitConfigBool("core.ignorecase"):
2187            depot_path = depot_path.lower()
2188
2189        if depot_path in self.client_spec_path_cache:
2190            return self.client_spec_path_cache[depot_path]
2191
2192        die( "Error: %s is not found in client spec path" % depot_path )
2193        return ""
2194
2195class P4Sync(Command, P4UserMap):
2196    delete_actions = ( "delete", "move/delete", "purge" )
2197
2198    def __init__(self):
2199        Command.__init__(self)
2200        P4UserMap.__init__(self)
2201        self.options = [
2202                optparse.make_option("--branch", dest="branch"),
2203                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2204                optparse.make_option("--changesfile", dest="changesFile"),
2205                optparse.make_option("--silent", dest="silent", action="store_true"),
2206                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2207                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2208                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2209                                     help="Import into refs/heads/ , not refs/remotes"),
2210                optparse.make_option("--max-changes", dest="maxChanges",
2211                                     help="Maximum number of changes to import"),
2212                optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2213                                     help="Internal block size to use when iteratively calling p4 changes"),
2214                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2215                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2216                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2217                                     help="Only sync files that are included in the Perforce Client Spec"),
2218                optparse.make_option("-/", dest="cloneExclude",
2219                                     action="append", type="string",
2220                                     help="exclude depot path"),
2221        ]
2222        self.description = """Imports from Perforce into a git repository.\n
2223    example:
2224    //depot/my/project/ -- to import the current head
2225    //depot/my/project/@all -- to import everything
2226    //depot/my/project/@1,6 -- to import only from revision 1 to 6
2227
2228    (a ... is not needed in the path p4 specification, it's added implicitly)"""
2229
2230        self.usage += " //depot/path[@revRange]"
2231        self.silent = False
2232        self.createdBranches = set()
2233        self.committedChanges = set()
2234        self.branch = ""
2235        self.detectBranches = False
2236        self.detectLabels = False
2237        self.importLabels = False
2238        self.changesFile = ""
2239        self.syncWithOrigin = True
2240        self.importIntoRemotes = True
2241        self.maxChanges = ""
2242        self.changes_block_size = None
2243        self.keepRepoPath = False
2244        self.depotPaths = None
2245        self.p4BranchesInGit = []
2246        self.cloneExclude = []
2247        self.useClientSpec = False
2248        self.useClientSpec_from_options = False
2249        self.clientSpecDirs = None
2250        self.tempBranches = []
2251        self.tempBranchLocation = "git-p4-tmp"
2252        self.largeFileSystem = None
2253
2254        if gitConfig('git-p4.largeFileSystem'):
2255            largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2256            self.largeFileSystem = largeFileSystemConstructor(
2257                lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2258            )
2259
2260        if gitConfig("git-p4.syncFromOrigin") == "false":
2261            self.syncWithOrigin = False
2262
2263    # This is required for the "append" cloneExclude action
2264    def ensure_value(self, attr, value):
2265        if not hasattr(self, attr) or getattr(self, attr) is None:
2266            setattr(self, attr, value)
2267        return getattr(self, attr)
2268
2269    # Force a checkpoint in fast-import and wait for it to finish
2270    def checkpoint(self):
2271        self.gitStream.write("checkpoint\n\n")
2272        self.gitStream.write("progress checkpoint\n\n")
2273        out = self.gitOutput.readline()
2274        if self.verbose:
2275            print "checkpoint finished: " + out
2276
2277    def extractFilesFromCommit(self, commit):
2278        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2279                             for path in self.cloneExclude]
2280        files = []
2281        fnum = 0
2282        while commit.has_key("depotFile%s" % fnum):
2283            path =  commit["depotFile%s" % fnum]
2284
2285            if [p for p in self.cloneExclude
2286                if p4PathStartsWith(path, p)]:
2287                found = False
2288            else:
2289                found = [p for p in self.depotPaths
2290                         if p4PathStartsWith(path, p)]
2291            if not found:
2292                fnum = fnum + 1
2293                continue
2294
2295            file = {}
2296            file["path"] = path
2297            file["rev"] = commit["rev%s" % fnum]
2298            file["action"] = commit["action%s" % fnum]
2299            file["type"] = commit["type%s" % fnum]
2300            files.append(file)
2301            fnum = fnum + 1
2302        return files
2303
2304    def stripRepoPath(self, path, prefixes):
2305        """When streaming files, this is called to map a p4 depot path
2306           to where it should go in git.  The prefixes are either
2307           self.depotPaths, or self.branchPrefixes in the case of
2308           branch detection."""
2309
2310        if self.useClientSpec:
2311            # branch detection moves files up a level (the branch name)
2312            # from what client spec interpretation gives
2313            path = self.clientSpecDirs.map_in_client(path)
2314            if self.detectBranches:
2315                for b in self.knownBranches:
2316                    if path.startswith(b + "/"):
2317                        path = path[len(b)+1:]
2318
2319        elif self.keepRepoPath:
2320            # Preserve everything in relative path name except leading
2321            # //depot/; just look at first prefix as they all should
2322            # be in the same depot.
2323            depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2324            if p4PathStartsWith(path, depot):
2325                path = path[len(depot):]
2326
2327        else:
2328            for p in prefixes:
2329                if p4PathStartsWith(path, p):
2330                    path = path[len(p):]
2331                    break
2332
2333        path = wildcard_decode(path)
2334        return path
2335
2336    def splitFilesIntoBranches(self, commit):
2337        """Look at each depotFile in the commit to figure out to what
2338           branch it belongs."""
2339
2340        if self.clientSpecDirs:
2341            files = self.extractFilesFromCommit(commit)
2342            self.clientSpecDirs.update_client_spec_path_cache(files)
2343
2344        branches = {}
2345        fnum = 0
2346        while commit.has_key("depotFile%s" % fnum):
2347            path =  commit["depotFile%s" % fnum]
2348            found = [p for p in self.depotPaths
2349                     if p4PathStartsWith(path, p)]
2350            if not found:
2351                fnum = fnum + 1
2352                continue
2353
2354            file = {}
2355            file["path"] = path
2356            file["rev"] = commit["rev%s" % fnum]
2357            file["action"] = commit["action%s" % fnum]
2358            file["type"] = commit["type%s" % fnum]
2359            fnum = fnum + 1
2360
2361            # start with the full relative path where this file would
2362            # go in a p4 client
2363            if self.useClientSpec:
2364                relPath = self.clientSpecDirs.map_in_client(path)
2365            else:
2366                relPath = self.stripRepoPath(path, self.depotPaths)
2367
2368            for branch in self.knownBranches.keys():
2369                # add a trailing slash so that a commit into qt/4.2foo
2370                # doesn't end up in qt/4.2, e.g.
2371                if relPath.startswith(branch + "/"):
2372                    if branch not in branches:
2373                        branches[branch] = []
2374                    branches[branch].append(file)
2375                    break
2376
2377        return branches
2378
2379    def writeToGitStream(self, gitMode, relPath, contents):
2380        self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2381        self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2382        for d in contents:
2383            self.gitStream.write(d)
2384        self.gitStream.write('\n')
2385
2386    # output one file from the P4 stream
2387    # - helper for streamP4Files
2388
2389    def streamOneP4File(self, file, contents):
2390        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2391        if verbose:
2392            size = int(self.stream_file['fileSize'])
2393            sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2394            sys.stdout.flush()
2395
2396        (type_base, type_mods) = split_p4_type(file["type"])
2397
2398        git_mode = "100644"
2399        if "x" in type_mods:
2400            git_mode = "100755"
2401        if type_base == "symlink":
2402            git_mode = "120000"
2403            # p4 print on a symlink sometimes contains "target\n";
2404            # if it does, remove the newline
2405            data = ''.join(contents)
2406            if not data:
2407                # Some version of p4 allowed creating a symlink that pointed
2408                # to nothing.  This causes p4 errors when checking out such
2409                # a change, and errors here too.  Work around it by ignoring
2410                # the bad symlink; hopefully a future change fixes it.
2411                print "\nIgnoring empty symlink in %s" % file['depotFile']
2412                return
2413            elif data[-1] == '\n':
2414                contents = [data[:-1]]
2415            else:
2416                contents = [data]
2417
2418        if type_base == "utf16":
2419            # p4 delivers different text in the python output to -G
2420            # than it does when using "print -o", or normal p4 client
2421            # operations.  utf16 is converted to ascii or utf8, perhaps.
2422            # But ascii text saved as -t utf16 is completely mangled.
2423            # Invoke print -o to get the real contents.
2424            #
2425            # On windows, the newlines will always be mangled by print, so put
2426            # them back too.  This is not needed to the cygwin windows version,
2427            # just the native "NT" type.
2428            #
2429            try:
2430                text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2431            except Exception as e:
2432                if 'Translation of file content failed' in str(e):
2433                    type_base = 'binary'
2434                else:
2435                    raise e
2436            else:
2437                if p4_version_string().find('/NT') >= 0:
2438                    text = text.replace('\r\n', '\n')
2439                contents = [ text ]
2440
2441        if type_base == "apple":
2442            # Apple filetype files will be streamed as a concatenation of
2443            # its appledouble header and the contents.  This is useless
2444            # on both macs and non-macs.  If using "print -q -o xx", it
2445            # will create "xx" with the data, and "%xx" with the header.
2446            # This is also not very useful.
2447            #
2448            # Ideally, someday, this script can learn how to generate
2449            # appledouble files directly and import those to git, but
2450            # non-mac machines can never find a use for apple filetype.
2451            print "\nIgnoring apple filetype file %s" % file['depotFile']
2452            return
2453
2454        # Note that we do not try to de-mangle keywords on utf16 files,
2455        # even though in theory somebody may want that.
2456        pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2457        if pattern:
2458            regexp = re.compile(pattern, re.VERBOSE)
2459            text = ''.join(contents)
2460            text = regexp.sub(r'$\1$', text)
2461            contents = [ text ]
2462
2463        try:
2464            relPath.decode('ascii')
2465        except:
2466            encoding = 'utf8'
2467            if gitConfig('git-p4.pathEncoding'):
2468                encoding = gitConfig('git-p4.pathEncoding')
2469            relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2470            if self.verbose:
2471                print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2472
2473        if self.largeFileSystem:
2474            (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2475
2476        self.writeToGitStream(git_mode, relPath, contents)
2477
2478    def streamOneP4Deletion(self, file):
2479        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2480        if verbose:
2481            sys.stdout.write("delete %s\n" % relPath)
2482            sys.stdout.flush()
2483        self.gitStream.write("D %s\n" % relPath)
2484
2485        if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2486            self.largeFileSystem.removeLargeFile(relPath)
2487
2488    # handle another chunk of streaming data
2489    def streamP4FilesCb(self, marshalled):
2490
2491        # catch p4 errors and complain
2492        err = None
2493        if "code" in marshalled:
2494            if marshalled["code"] == "error":
2495                if "data" in marshalled:
2496                    err = marshalled["data"].rstrip()
2497
2498        if not err and 'fileSize' in self.stream_file:
2499            required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2500            if required_bytes > 0:
2501                err = 'Not enough space left on %s! Free at least %i MB.' % (
2502                    os.getcwd(), required_bytes/1024/1024
2503                )
2504
2505        if err:
2506            f = None
2507            if self.stream_have_file_info:
2508                if "depotFile" in self.stream_file:
2509                    f = self.stream_file["depotFile"]
2510            # force a failure in fast-import, else an empty
2511            # commit will be made
2512            self.gitStream.write("\n")
2513            self.gitStream.write("die-now\n")
2514            self.gitStream.close()
2515            # ignore errors, but make sure it exits first
2516            self.importProcess.wait()
2517            if f:
2518                die("Error from p4 print for %s: %s" % (f, err))
2519            else:
2520                die("Error from p4 print: %s" % err)
2521
2522        if marshalled.has_key('depotFile') and self.stream_have_file_info:
2523            # start of a new file - output the old one first
2524            self.streamOneP4File(self.stream_file, self.stream_contents)
2525            self.stream_file = {}
2526            self.stream_contents = []
2527            self.stream_have_file_info = False
2528
2529        # pick up the new file information... for the
2530        # 'data' field we need to append to our array
2531        for k in marshalled.keys():
2532            if k == 'data':
2533                if 'streamContentSize' not in self.stream_file:
2534                    self.stream_file['streamContentSize'] = 0
2535                self.stream_file['streamContentSize'] += len(marshalled['data'])
2536                self.stream_contents.append(marshalled['data'])
2537            else:
2538                self.stream_file[k] = marshalled[k]
2539
2540        if (verbose and
2541            'streamContentSize' in self.stream_file and
2542            'fileSize' in self.stream_file and
2543            'depotFile' in self.stream_file):
2544            size = int(self.stream_file["fileSize"])
2545            if size > 0:
2546                progress = 100*self.stream_file['streamContentSize']/size
2547                sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2548                sys.stdout.flush()
2549
2550        self.stream_have_file_info = True
2551
2552    # Stream directly from "p4 files" into "git fast-import"
2553    def streamP4Files(self, files):
2554        filesForCommit = []
2555        filesToRead = []
2556        filesToDelete = []
2557
2558        for f in files:
2559            # if using a client spec, only add the files that have
2560            # a path in the client
2561            if self.clientSpecDirs:
2562                if self.clientSpecDirs.map_in_client(f['path']) == "":
2563                    continue
2564
2565            filesForCommit.append(f)
2566            if f['action'] in self.delete_actions:
2567                filesToDelete.append(f)
2568            else:
2569                filesToRead.append(f)
2570
2571        # deleted files...
2572        for f in filesToDelete:
2573            self.streamOneP4Deletion(f)
2574
2575        if len(filesToRead) > 0:
2576            self.stream_file = {}
2577            self.stream_contents = []
2578            self.stream_have_file_info = False
2579
2580            # curry self argument
2581            def streamP4FilesCbSelf(entry):
2582                self.streamP4FilesCb(entry)
2583
2584            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2585
2586            p4CmdList(["-x", "-", "print"],
2587                      stdin=fileArgs,
2588                      cb=streamP4FilesCbSelf)
2589
2590            # do the last chunk
2591            if self.stream_file.has_key('depotFile'):
2592                self.streamOneP4File(self.stream_file, self.stream_contents)
2593
2594    def make_email(self, userid):
2595        if userid in self.users:
2596            return self.users[userid]
2597        else:
2598            return "%s <a@b>" % userid
2599
2600    def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2601        """ Stream a p4 tag.
2602        commit is either a git commit, or a fast-import mark, ":<p4commit>"
2603        """
2604
2605        if verbose:
2606            print "writing tag %s for commit %s" % (labelName, commit)
2607        gitStream.write("tag %s\n" % labelName)
2608        gitStream.write("from %s\n" % commit)
2609
2610        if labelDetails.has_key('Owner'):
2611            owner = labelDetails["Owner"]
2612        else:
2613            owner = None
2614
2615        # Try to use the owner of the p4 label, or failing that,
2616        # the current p4 user id.
2617        if owner:
2618            email = self.make_email(owner)
2619        else:
2620            email = self.make_email(self.p4UserId())
2621        tagger = "%s %s %s" % (email, epoch, self.tz)
2622
2623        gitStream.write("tagger %s\n" % tagger)
2624
2625        print "labelDetails=",labelDetails
2626        if labelDetails.has_key('Description'):
2627            description = labelDetails['Description']
2628        else:
2629            description = 'Label from git p4'
2630
2631        gitStream.write("data %d\n" % len(description))
2632        gitStream.write(description)
2633        gitStream.write("\n")
2634
2635    def commit(self, details, files, branch, parent = ""):
2636        epoch = details["time"]
2637        author = details["user"]
2638
2639        if self.verbose:
2640            print "commit into %s" % branch
2641
2642        # start with reading files; if that fails, we should not
2643        # create a commit.
2644        new_files = []
2645        for f in files:
2646            if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2647                new_files.append (f)
2648            else:
2649                sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2650
2651        if self.clientSpecDirs:
2652            self.clientSpecDirs.update_client_spec_path_cache(files)
2653
2654        self.gitStream.write("commit %s\n" % branch)
2655        self.gitStream.write("mark :%s\n" % details["change"])
2656        self.committedChanges.add(int(details["change"]))
2657        committer = ""
2658        if author not in self.users:
2659            self.getUserMapFromPerforceServer()
2660        committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2661
2662        self.gitStream.write("committer %s\n" % committer)
2663
2664        self.gitStream.write("data <<EOT\n")
2665        self.gitStream.write(details["desc"])
2666        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2667                             (','.join(self.branchPrefixes), details["change"]))
2668        if len(details['options']) > 0:
2669            self.gitStream.write(": options = %s" % details['options'])
2670        self.gitStream.write("]\nEOT\n\n")
2671
2672        if len(parent) > 0:
2673            if self.verbose:
2674                print "parent %s" % parent
2675            self.gitStream.write("from %s\n" % parent)
2676
2677        self.streamP4Files(new_files)
2678        self.gitStream.write("\n")
2679
2680        change = int(details["change"])
2681
2682        if self.labels.has_key(change):
2683            label = self.labels[change]
2684            labelDetails = label[0]
2685            labelRevisions = label[1]
2686            if self.verbose:
2687                print "Change %s is labelled %s" % (change, labelDetails)
2688
2689            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2690                                                for p in self.branchPrefixes])
2691
2692            if len(files) == len(labelRevisions):
2693
2694                cleanedFiles = {}
2695                for info in files:
2696                    if info["action"] in self.delete_actions:
2697                        continue
2698                    cleanedFiles[info["depotFile"]] = info["rev"]
2699
2700                if cleanedFiles == labelRevisions:
2701                    self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2702
2703                else:
2704                    if not self.silent:
2705                        print ("Tag %s does not match with change %s: files do not match."
2706                               % (labelDetails["label"], change))
2707
2708            else:
2709                if not self.silent:
2710                    print ("Tag %s does not match with change %s: file count is different."
2711                           % (labelDetails["label"], change))
2712
2713    # Build a dictionary of changelists and labels, for "detect-labels" option.
2714    def getLabels(self):
2715        self.labels = {}
2716
2717        l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2718        if len(l) > 0 and not self.silent:
2719            print "Finding files belonging to labels in %s" % `self.depotPaths`
2720
2721        for output in l:
2722            label = output["label"]
2723            revisions = {}
2724            newestChange = 0
2725            if self.verbose:
2726                print "Querying files for label %s" % label
2727            for file in p4CmdList(["files"] +
2728                                      ["%s...@%s" % (p, label)
2729                                          for p in self.depotPaths]):
2730                revisions[file["depotFile"]] = file["rev"]
2731                change = int(file["change"])
2732                if change > newestChange:
2733                    newestChange = change
2734
2735            self.labels[newestChange] = [output, revisions]
2736
2737        if self.verbose:
2738            print "Label changes: %s" % self.labels.keys()
2739
2740    # Import p4 labels as git tags. A direct mapping does not
2741    # exist, so assume that if all the files are at the same revision
2742    # then we can use that, or it's something more complicated we should
2743    # just ignore.
2744    def importP4Labels(self, stream, p4Labels):
2745        if verbose:
2746            print "import p4 labels: " + ' '.join(p4Labels)
2747
2748        ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2749        validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2750        if len(validLabelRegexp) == 0:
2751            validLabelRegexp = defaultLabelRegexp
2752        m = re.compile(validLabelRegexp)
2753
2754        for name in p4Labels:
2755            commitFound = False
2756
2757            if not m.match(name):
2758                if verbose:
2759                    print "label %s does not match regexp %s" % (name,validLabelRegexp)
2760                continue
2761
2762            if name in ignoredP4Labels:
2763                continue
2764
2765            labelDetails = p4CmdList(['label', "-o", name])[0]
2766
2767            # get the most recent changelist for each file in this label
2768            change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2769                                for p in self.depotPaths])
2770
2771            if change.has_key('change'):
2772                # find the corresponding git commit; take the oldest commit
2773                changelist = int(change['change'])
2774                if changelist in self.committedChanges:
2775                    gitCommit = ":%d" % changelist       # use a fast-import mark
2776                    commitFound = True
2777                else:
2778                    gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2779                        "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2780                    if len(gitCommit) == 0:
2781                        print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2782                    else:
2783                        commitFound = True
2784                        gitCommit = gitCommit.strip()
2785
2786                if commitFound:
2787                    # Convert from p4 time format
2788                    try:
2789                        tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2790                    except ValueError:
2791                        print "Could not convert label time %s" % labelDetails['Update']
2792                        tmwhen = 1
2793
2794                    when = int(time.mktime(tmwhen))
2795                    self.streamTag(stream, name, labelDetails, gitCommit, when)
2796                    if verbose:
2797                        print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2798            else:
2799                if verbose:
2800                    print "Label %s has no changelists - possibly deleted?" % name
2801
2802            if not commitFound:
2803                # We can't import this label; don't try again as it will get very
2804                # expensive repeatedly fetching all the files for labels that will
2805                # never be imported. If the label is moved in the future, the
2806                # ignore will need to be removed manually.
2807                system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2808
2809    def guessProjectName(self):
2810        for p in self.depotPaths:
2811            if p.endswith("/"):
2812                p = p[:-1]
2813            p = p[p.strip().rfind("/") + 1:]
2814            if not p.endswith("/"):
2815               p += "/"
2816            return p
2817
2818    def getBranchMapping(self):
2819        lostAndFoundBranches = set()
2820
2821        user = gitConfig("git-p4.branchUser")
2822        if len(user) > 0:
2823            command = "branches -u %s" % user
2824        else:
2825            command = "branches"
2826
2827        for info in p4CmdList(command):
2828            details = p4Cmd(["branch", "-o", info["branch"]])
2829            viewIdx = 0
2830            while details.has_key("View%s" % viewIdx):
2831                paths = details["View%s" % viewIdx].split(" ")
2832                viewIdx = viewIdx + 1
2833                # require standard //depot/foo/... //depot/bar/... mapping
2834                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2835                    continue
2836                source = paths[0]
2837                destination = paths[1]
2838                ## HACK
2839                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2840                    source = source[len(self.depotPaths[0]):-4]
2841                    destination = destination[len(self.depotPaths[0]):-4]
2842
2843                    if destination in self.knownBranches:
2844                        if not self.silent:
2845                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2846                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2847                        continue
2848
2849                    self.knownBranches[destination] = source
2850
2851                    lostAndFoundBranches.discard(destination)
2852
2853                    if source not in self.knownBranches:
2854                        lostAndFoundBranches.add(source)
2855
2856        # Perforce does not strictly require branches to be defined, so we also
2857        # check git config for a branch list.
2858        #
2859        # Example of branch definition in git config file:
2860        # [git-p4]
2861        #   branchList=main:branchA
2862        #   branchList=main:branchB
2863        #   branchList=branchA:branchC
2864        configBranches = gitConfigList("git-p4.branchList")
2865        for branch in configBranches:
2866            if branch:
2867                (source, destination) = branch.split(":")
2868                self.knownBranches[destination] = source
2869
2870                lostAndFoundBranches.discard(destination)
2871
2872                if source not in self.knownBranches:
2873                    lostAndFoundBranches.add(source)
2874
2875
2876        for branch in lostAndFoundBranches:
2877            self.knownBranches[branch] = branch
2878
2879    def getBranchMappingFromGitBranches(self):
2880        branches = p4BranchesInGit(self.importIntoRemotes)
2881        for branch in branches.keys():
2882            if branch == "master":
2883                branch = "main"
2884            else:
2885                branch = branch[len(self.projectName):]
2886            self.knownBranches[branch] = branch
2887
2888    def updateOptionDict(self, d):
2889        option_keys = {}
2890        if self.keepRepoPath:
2891            option_keys['keepRepoPath'] = 1
2892
2893        d["options"] = ' '.join(sorted(option_keys.keys()))
2894
2895    def readOptions(self, d):
2896        self.keepRepoPath = (d.has_key('options')
2897                             and ('keepRepoPath' in d['options']))
2898
2899    def gitRefForBranch(self, branch):
2900        if branch == "main":
2901            return self.refPrefix + "master"
2902
2903        if len(branch) <= 0:
2904            return branch
2905
2906        return self.refPrefix + self.projectName + branch
2907
2908    def gitCommitByP4Change(self, ref, change):
2909        if self.verbose:
2910            print "looking in ref " + ref + " for change %s using bisect..." % change
2911
2912        earliestCommit = ""
2913        latestCommit = parseRevision(ref)
2914
2915        while True:
2916            if self.verbose:
2917                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2918            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2919            if len(next) == 0:
2920                if self.verbose:
2921                    print "argh"
2922                return ""
2923            log = extractLogMessageFromGitCommit(next)
2924            settings = extractSettingsGitLog(log)
2925            currentChange = int(settings['change'])
2926            if self.verbose:
2927                print "current change %s" % currentChange
2928
2929            if currentChange == change:
2930                if self.verbose:
2931                    print "found %s" % next
2932                return next
2933
2934            if currentChange < change:
2935                earliestCommit = "^%s" % next
2936            else:
2937                latestCommit = "%s" % next
2938
2939        return ""
2940
2941    def importNewBranch(self, branch, maxChange):
2942        # make fast-import flush all changes to disk and update the refs using the checkpoint
2943        # command so that we can try to find the branch parent in the git history
2944        self.gitStream.write("checkpoint\n\n");
2945        self.gitStream.flush();
2946        branchPrefix = self.depotPaths[0] + branch + "/"
2947        range = "@1,%s" % maxChange
2948        #print "prefix" + branchPrefix
2949        changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2950        if len(changes) <= 0:
2951            return False
2952        firstChange = changes[0]
2953        #print "first change in branch: %s" % firstChange
2954        sourceBranch = self.knownBranches[branch]
2955        sourceDepotPath = self.depotPaths[0] + sourceBranch
2956        sourceRef = self.gitRefForBranch(sourceBranch)
2957        #print "source " + sourceBranch
2958
2959        branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2960        #print "branch parent: %s" % branchParentChange
2961        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2962        if len(gitParent) > 0:
2963            self.initialParents[self.gitRefForBranch(branch)] = gitParent
2964            #print "parent git commit: %s" % gitParent
2965
2966        self.importChanges(changes)
2967        return True
2968
2969    def searchParent(self, parent, branch, target):
2970        parentFound = False
2971        for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2972                                     "--no-merges", parent]):
2973            blob = blob.strip()
2974            if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2975                parentFound = True
2976                if self.verbose:
2977                    print "Found parent of %s in commit %s" % (branch, blob)
2978                break
2979        if parentFound:
2980            return blob
2981        else:
2982            return None
2983
2984    def importChanges(self, changes):
2985        cnt = 1
2986        for change in changes:
2987            description = p4_describe(change)
2988            self.updateOptionDict(description)
2989
2990            if not self.silent:
2991                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2992                sys.stdout.flush()
2993            cnt = cnt + 1
2994
2995            try:
2996                if self.detectBranches:
2997                    branches = self.splitFilesIntoBranches(description)
2998                    for branch in branches.keys():
2999                        ## HACK  --hwn
3000                        branchPrefix = self.depotPaths[0] + branch + "/"
3001                        self.branchPrefixes = [ branchPrefix ]
3002
3003                        parent = ""
3004
3005                        filesForCommit = branches[branch]
3006
3007                        if self.verbose:
3008                            print "branch is %s" % branch
3009
3010                        self.updatedBranches.add(branch)
3011
3012                        if branch not in self.createdBranches:
3013                            self.createdBranches.add(branch)
3014                            parent = self.knownBranches[branch]
3015                            if parent == branch:
3016                                parent = ""
3017                            else:
3018                                fullBranch = self.projectName + branch
3019                                if fullBranch not in self.p4BranchesInGit:
3020                                    if not self.silent:
3021                                        print("\n    Importing new branch %s" % fullBranch);
3022                                    if self.importNewBranch(branch, change - 1):
3023                                        parent = ""
3024                                        self.p4BranchesInGit.append(fullBranch)
3025                                    if not self.silent:
3026                                        print("\n    Resuming with change %s" % change);
3027
3028                                if self.verbose:
3029                                    print "parent determined through known branches: %s" % parent
3030
3031                        branch = self.gitRefForBranch(branch)
3032                        parent = self.gitRefForBranch(parent)
3033
3034                        if self.verbose:
3035                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3036
3037                        if len(parent) == 0 and branch in self.initialParents:
3038                            parent = self.initialParents[branch]
3039                            del self.initialParents[branch]
3040
3041                        blob = None
3042                        if len(parent) > 0:
3043                            tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3044                            if self.verbose:
3045                                print "Creating temporary branch: " + tempBranch
3046                            self.commit(description, filesForCommit, tempBranch)
3047                            self.tempBranches.append(tempBranch)
3048                            self.checkpoint()
3049                            blob = self.searchParent(parent, branch, tempBranch)
3050                        if blob:
3051                            self.commit(description, filesForCommit, branch, blob)
3052                        else:
3053                            if self.verbose:
3054                                print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3055                            self.commit(description, filesForCommit, branch, parent)
3056                else:
3057                    files = self.extractFilesFromCommit(description)
3058                    self.commit(description, files, self.branch,
3059                                self.initialParent)
3060                    # only needed once, to connect to the previous commit
3061                    self.initialParent = ""
3062            except IOError:
3063                print self.gitError.read()
3064                sys.exit(1)
3065
3066    def importHeadRevision(self, revision):
3067        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3068
3069        details = {}
3070        details["user"] = "git perforce import user"
3071        details["desc"] = ("Initial import of %s from the state at revision %s\n"
3072                           % (' '.join(self.depotPaths), revision))
3073        details["change"] = revision
3074        newestRevision = 0
3075
3076        fileCnt = 0
3077        fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3078
3079        for info in p4CmdList(["files"] + fileArgs):
3080
3081            if 'code' in info and info['code'] == 'error':
3082                sys.stderr.write("p4 returned an error: %s\n"
3083                                 % info['data'])
3084                if info['data'].find("must refer to client") >= 0:
3085                    sys.stderr.write("This particular p4 error is misleading.\n")
3086                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
3087                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3088                sys.exit(1)
3089            if 'p4ExitCode' in info:
3090                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3091                sys.exit(1)
3092
3093
3094            change = int(info["change"])
3095            if change > newestRevision:
3096                newestRevision = change
3097
3098            if info["action"] in self.delete_actions:
3099                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3100                #fileCnt = fileCnt + 1
3101                continue
3102
3103            for prop in ["depotFile", "rev", "action", "type" ]:
3104                details["%s%s" % (prop, fileCnt)] = info[prop]
3105
3106            fileCnt = fileCnt + 1
3107
3108        details["change"] = newestRevision
3109
3110        # Use time from top-most change so that all git p4 clones of
3111        # the same p4 repo have the same commit SHA1s.
3112        res = p4_describe(newestRevision)
3113        details["time"] = res["time"]
3114
3115        self.updateOptionDict(details)
3116        try:
3117            self.commit(details, self.extractFilesFromCommit(details), self.branch)
3118        except IOError:
3119            print "IO error with git fast-import. Is your git version recent enough?"
3120            print self.gitError.read()
3121
3122
3123    def run(self, args):
3124        self.depotPaths = []
3125        self.changeRange = ""
3126        self.previousDepotPaths = []
3127        self.hasOrigin = False
3128
3129        # map from branch depot path to parent branch
3130        self.knownBranches = {}
3131        self.initialParents = {}
3132
3133        if self.importIntoRemotes:
3134            self.refPrefix = "refs/remotes/p4/"
3135        else:
3136            self.refPrefix = "refs/heads/p4/"
3137
3138        if self.syncWithOrigin:
3139            self.hasOrigin = originP4BranchesExist()
3140            if self.hasOrigin:
3141                if not self.silent:
3142                    print 'Syncing with origin first, using "git fetch origin"'
3143                system("git fetch origin")
3144
3145        branch_arg_given = bool(self.branch)
3146        if len(self.branch) == 0:
3147            self.branch = self.refPrefix + "master"
3148            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3149                system("git update-ref %s refs/heads/p4" % self.branch)
3150                system("git branch -D p4")
3151
3152        # accept either the command-line option, or the configuration variable
3153        if self.useClientSpec:
3154            # will use this after clone to set the variable
3155            self.useClientSpec_from_options = True
3156        else:
3157            if gitConfigBool("git-p4.useclientspec"):
3158                self.useClientSpec = True
3159        if self.useClientSpec:
3160            self.clientSpecDirs = getClientSpec()
3161
3162        # TODO: should always look at previous commits,
3163        # merge with previous imports, if possible.
3164        if args == []:
3165            if self.hasOrigin:
3166                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3167
3168            # branches holds mapping from branch name to sha1
3169            branches = p4BranchesInGit(self.importIntoRemotes)
3170
3171            # restrict to just this one, disabling detect-branches
3172            if branch_arg_given:
3173                short = self.branch.split("/")[-1]
3174                if short in branches:
3175                    self.p4BranchesInGit = [ short ]
3176            else:
3177                self.p4BranchesInGit = branches.keys()
3178
3179            if len(self.p4BranchesInGit) > 1:
3180                if not self.silent:
3181                    print "Importing from/into multiple branches"
3182                self.detectBranches = True
3183                for branch in branches.keys():
3184                    self.initialParents[self.refPrefix + branch] = \
3185                        branches[branch]
3186
3187            if self.verbose:
3188                print "branches: %s" % self.p4BranchesInGit
3189
3190            p4Change = 0
3191            for branch in self.p4BranchesInGit:
3192                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3193
3194                settings = extractSettingsGitLog(logMsg)
3195
3196                self.readOptions(settings)
3197                if (settings.has_key('depot-paths')
3198                    and settings.has_key ('change')):
3199                    change = int(settings['change']) + 1
3200                    p4Change = max(p4Change, change)
3201
3202                    depotPaths = sorted(settings['depot-paths'])
3203                    if self.previousDepotPaths == []:
3204                        self.previousDepotPaths = depotPaths
3205                    else:
3206                        paths = []
3207                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3208                            prev_list = prev.split("/")
3209                            cur_list = cur.split("/")
3210                            for i in range(0, min(len(cur_list), len(prev_list))):
3211                                if cur_list[i] <> prev_list[i]:
3212                                    i = i - 1
3213                                    break
3214
3215                            paths.append ("/".join(cur_list[:i + 1]))
3216
3217                        self.previousDepotPaths = paths
3218
3219            if p4Change > 0:
3220                self.depotPaths = sorted(self.previousDepotPaths)
3221                self.changeRange = "@%s,#head" % p4Change
3222                if not self.silent and not self.detectBranches:
3223                    print "Performing incremental import into %s git branch" % self.branch
3224
3225        # accept multiple ref name abbreviations:
3226        #    refs/foo/bar/branch -> use it exactly
3227        #    p4/branch -> prepend refs/remotes/ or refs/heads/
3228        #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3229        if not self.branch.startswith("refs/"):
3230            if self.importIntoRemotes:
3231                prepend = "refs/remotes/"
3232            else:
3233                prepend = "refs/heads/"
3234            if not self.branch.startswith("p4/"):
3235                prepend += "p4/"
3236            self.branch = prepend + self.branch
3237
3238        if len(args) == 0 and self.depotPaths:
3239            if not self.silent:
3240                print "Depot paths: %s" % ' '.join(self.depotPaths)
3241        else:
3242            if self.depotPaths and self.depotPaths != args:
3243                print ("previous import used depot path %s and now %s was specified. "
3244                       "This doesn't work!" % (' '.join (self.depotPaths),
3245                                               ' '.join (args)))
3246                sys.exit(1)
3247
3248            self.depotPaths = sorted(args)
3249
3250        revision = ""
3251        self.users = {}
3252
3253        # Make sure no revision specifiers are used when --changesfile
3254        # is specified.
3255        bad_changesfile = False
3256        if len(self.changesFile) > 0:
3257            for p in self.depotPaths:
3258                if p.find("@") >= 0 or p.find("#") >= 0:
3259                    bad_changesfile = True
3260                    break
3261        if bad_changesfile:
3262            die("Option --changesfile is incompatible with revision specifiers")
3263
3264        newPaths = []
3265        for p in self.depotPaths:
3266            if p.find("@") != -1:
3267                atIdx = p.index("@")
3268                self.changeRange = p[atIdx:]
3269                if self.changeRange == "@all":
3270                    self.changeRange = ""
3271                elif ',' not in self.changeRange:
3272                    revision = self.changeRange
3273                    self.changeRange = ""
3274                p = p[:atIdx]
3275            elif p.find("#") != -1:
3276                hashIdx = p.index("#")
3277                revision = p[hashIdx:]
3278                p = p[:hashIdx]
3279            elif self.previousDepotPaths == []:
3280                # pay attention to changesfile, if given, else import
3281                # the entire p4 tree at the head revision
3282                if len(self.changesFile) == 0:
3283                    revision = "#head"
3284
3285            p = re.sub ("\.\.\.$", "", p)
3286            if not p.endswith("/"):
3287                p += "/"
3288
3289            newPaths.append(p)
3290
3291        self.depotPaths = newPaths
3292
3293        # --detect-branches may change this for each branch
3294        self.branchPrefixes = self.depotPaths
3295
3296        self.loadUserMapFromCache()
3297        self.labels = {}
3298        if self.detectLabels:
3299            self.getLabels();
3300
3301        if self.detectBranches:
3302            ## FIXME - what's a P4 projectName ?
3303            self.projectName = self.guessProjectName()
3304
3305            if self.hasOrigin:
3306                self.getBranchMappingFromGitBranches()
3307            else:
3308                self.getBranchMapping()
3309            if self.verbose:
3310                print "p4-git branches: %s" % self.p4BranchesInGit
3311                print "initial parents: %s" % self.initialParents
3312            for b in self.p4BranchesInGit:
3313                if b != "master":
3314
3315                    ## FIXME
3316                    b = b[len(self.projectName):]
3317                self.createdBranches.add(b)
3318
3319        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3320
3321        self.importProcess = subprocess.Popen(["git", "fast-import"],
3322                                              stdin=subprocess.PIPE,
3323                                              stdout=subprocess.PIPE,
3324                                              stderr=subprocess.PIPE);
3325        self.gitOutput = self.importProcess.stdout
3326        self.gitStream = self.importProcess.stdin
3327        self.gitError = self.importProcess.stderr
3328
3329        if revision:
3330            self.importHeadRevision(revision)
3331        else:
3332            changes = []
3333
3334            if len(self.changesFile) > 0:
3335                output = open(self.changesFile).readlines()
3336                changeSet = set()
3337                for line in output:
3338                    changeSet.add(int(line))
3339
3340                for change in changeSet:
3341                    changes.append(change)
3342
3343                changes.sort()
3344            else:
3345                # catch "git p4 sync" with no new branches, in a repo that
3346                # does not have any existing p4 branches
3347                if len(args) == 0:
3348                    if not self.p4BranchesInGit:
3349                        die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3350
3351                    # The default branch is master, unless --branch is used to
3352                    # specify something else.  Make sure it exists, or complain
3353                    # nicely about how to use --branch.
3354                    if not self.detectBranches:
3355                        if not branch_exists(self.branch):
3356                            if branch_arg_given:
3357                                die("Error: branch %s does not exist." % self.branch)
3358                            else:
3359                                die("Error: no branch %s; perhaps specify one with --branch." %
3360                                    self.branch)
3361
3362                if self.verbose:
3363                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3364                                                              self.changeRange)
3365                changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3366
3367                if len(self.maxChanges) > 0:
3368                    changes = changes[:min(int(self.maxChanges), len(changes))]
3369
3370            if len(changes) == 0:
3371                if not self.silent:
3372                    print "No changes to import!"
3373            else:
3374                if not self.silent and not self.detectBranches:
3375                    print "Import destination: %s" % self.branch
3376
3377                self.updatedBranches = set()
3378
3379                if not self.detectBranches:
3380                    if args:
3381                        # start a new branch
3382                        self.initialParent = ""
3383                    else:
3384                        # build on a previous revision
3385                        self.initialParent = parseRevision(self.branch)
3386
3387                self.importChanges(changes)
3388
3389                if not self.silent:
3390                    print ""
3391                    if len(self.updatedBranches) > 0:
3392                        sys.stdout.write("Updated branches: ")
3393                        for b in self.updatedBranches:
3394                            sys.stdout.write("%s " % b)
3395                        sys.stdout.write("\n")
3396
3397        if gitConfigBool("git-p4.importLabels"):
3398            self.importLabels = True
3399
3400        if self.importLabels:
3401            p4Labels = getP4Labels(self.depotPaths)
3402            gitTags = getGitTags()
3403
3404            missingP4Labels = p4Labels - gitTags
3405            self.importP4Labels(self.gitStream, missingP4Labels)
3406
3407        self.gitStream.close()
3408        if self.importProcess.wait() != 0:
3409            die("fast-import failed: %s" % self.gitError.read())
3410        self.gitOutput.close()
3411        self.gitError.close()
3412
3413        # Cleanup temporary branches created during import
3414        if self.tempBranches != []:
3415            for branch in self.tempBranches:
3416                read_pipe("git update-ref -d %s" % branch)
3417            os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3418
3419        # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3420        # a convenient shortcut refname "p4".
3421        if self.importIntoRemotes:
3422            head_ref = self.refPrefix + "HEAD"
3423            if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3424                system(["git", "symbolic-ref", head_ref, self.branch])
3425
3426        return True
3427
3428class P4Rebase(Command):
3429    def __init__(self):
3430        Command.__init__(self)
3431        self.options = [
3432                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3433        ]
3434        self.importLabels = False
3435        self.description = ("Fetches the latest revision from perforce and "
3436                            + "rebases the current work (branch) against it")
3437
3438    def run(self, args):
3439        sync = P4Sync()
3440        sync.importLabels = self.importLabels
3441        sync.run([])
3442
3443        return self.rebase()
3444
3445    def rebase(self):
3446        if os.system("git update-index --refresh") != 0:
3447            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3448        if len(read_pipe("git diff-index HEAD --")) > 0:
3449            die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3450
3451        [upstream, settings] = findUpstreamBranchPoint()
3452        if len(upstream) == 0:
3453            die("Cannot find upstream branchpoint for rebase")
3454
3455        # the branchpoint may be p4/foo~3, so strip off the parent
3456        upstream = re.sub("~[0-9]+$", "", upstream)
3457
3458        print "Rebasing the current branch onto %s" % upstream
3459        oldHead = read_pipe("git rev-parse HEAD").strip()
3460        system("git rebase %s" % upstream)
3461        system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3462        return True
3463
3464class P4Clone(P4Sync):
3465    def __init__(self):
3466        P4Sync.__init__(self)
3467        self.description = "Creates a new git repository and imports from Perforce into it"
3468        self.usage = "usage: %prog [options] //depot/path[@revRange]"
3469        self.options += [
3470            optparse.make_option("--destination", dest="cloneDestination",
3471                                 action='store', default=None,
3472                                 help="where to leave result of the clone"),
3473            optparse.make_option("--bare", dest="cloneBare",
3474                                 action="store_true", default=False),
3475        ]
3476        self.cloneDestination = None
3477        self.needsGit = False
3478        self.cloneBare = False
3479
3480    def defaultDestination(self, args):
3481        ## TODO: use common prefix of args?
3482        depotPath = args[0]
3483        depotDir = re.sub("(@[^@]*)$", "", depotPath)
3484        depotDir = re.sub("(#[^#]*)$", "", depotDir)
3485        depotDir = re.sub(r"\.\.\.$", "", depotDir)
3486        depotDir = re.sub(r"/$", "", depotDir)
3487        return os.path.split(depotDir)[1]
3488
3489    def run(self, args):
3490        if len(args) < 1:
3491            return False
3492
3493        if self.keepRepoPath and not self.cloneDestination:
3494            sys.stderr.write("Must specify destination for --keep-path\n")
3495            sys.exit(1)
3496
3497        depotPaths = args
3498
3499        if not self.cloneDestination and len(depotPaths) > 1:
3500            self.cloneDestination = depotPaths[-1]
3501            depotPaths = depotPaths[:-1]
3502
3503        self.cloneExclude = ["/"+p for p in self.cloneExclude]
3504        for p in depotPaths:
3505            if not p.startswith("//"):
3506                sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3507                return False
3508
3509        if not self.cloneDestination:
3510            self.cloneDestination = self.defaultDestination(args)
3511
3512        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3513
3514        if not os.path.exists(self.cloneDestination):
3515            os.makedirs(self.cloneDestination)
3516        chdir(self.cloneDestination)
3517
3518        init_cmd = [ "git", "init" ]
3519        if self.cloneBare:
3520            init_cmd.append("--bare")
3521        retcode = subprocess.call(init_cmd)
3522        if retcode:
3523            raise CalledProcessError(retcode, init_cmd)
3524
3525        if not P4Sync.run(self, depotPaths):
3526            return False
3527
3528        # create a master branch and check out a work tree
3529        if gitBranchExists(self.branch):
3530            system([ "git", "branch", "master", self.branch ])
3531            if not self.cloneBare:
3532                system([ "git", "checkout", "-f" ])
3533        else:
3534            print 'Not checking out any branch, use ' \
3535                  '"git checkout -q -b master <branch>"'
3536
3537        # auto-set this variable if invoked with --use-client-spec
3538        if self.useClientSpec_from_options:
3539            system("git config --bool git-p4.useclientspec true")
3540
3541        return True
3542
3543class P4Branches(Command):
3544    def __init__(self):
3545        Command.__init__(self)
3546        self.options = [ ]
3547        self.description = ("Shows the git branches that hold imports and their "
3548                            + "corresponding perforce depot paths")
3549        self.verbose = False
3550
3551    def run(self, args):
3552        if originP4BranchesExist():
3553            createOrUpdateBranchesFromOrigin()
3554
3555        cmdline = "git rev-parse --symbolic "
3556        cmdline += " --remotes"
3557
3558        for line in read_pipe_lines(cmdline):
3559            line = line.strip()
3560
3561            if not line.startswith('p4/') or line == "p4/HEAD":
3562                continue
3563            branch = line
3564
3565            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3566            settings = extractSettingsGitLog(log)
3567
3568            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3569        return True
3570
3571class HelpFormatter(optparse.IndentedHelpFormatter):
3572    def __init__(self):
3573        optparse.IndentedHelpFormatter.__init__(self)
3574
3575    def format_description(self, description):
3576        if description:
3577            return description + "\n"
3578        else:
3579            return ""
3580
3581def printUsage(commands):
3582    print "usage: %s <command> [options]" % sys.argv[0]
3583    print ""
3584    print "valid commands: %s" % ", ".join(commands)
3585    print ""
3586    print "Try %s <command> --help for command specific help." % sys.argv[0]
3587    print ""
3588
3589commands = {
3590    "debug" : P4Debug,
3591    "submit" : P4Submit,
3592    "commit" : P4Submit,
3593    "sync" : P4Sync,
3594    "rebase" : P4Rebase,
3595    "clone" : P4Clone,
3596    "rollback" : P4RollBack,
3597    "branches" : P4Branches
3598}
3599
3600
3601def main():
3602    if len(sys.argv[1:]) == 0:
3603        printUsage(commands.keys())
3604        sys.exit(2)
3605
3606    cmdName = sys.argv[1]
3607    try:
3608        klass = commands[cmdName]
3609        cmd = klass()
3610    except KeyError:
3611        print "unknown command %s" % cmdName
3612        print ""
3613        printUsage(commands.keys())
3614        sys.exit(2)
3615
3616    options = cmd.options
3617    cmd.gitdir = os.environ.get("GIT_DIR", None)
3618
3619    args = sys.argv[2:]
3620
3621    options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3622    if cmd.needsGit:
3623        options.append(optparse.make_option("--git-dir", dest="gitdir"))
3624
3625    parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3626                                   options,
3627                                   description = cmd.description,
3628                                   formatter = HelpFormatter())
3629
3630    (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3631    global verbose
3632    verbose = cmd.verbose
3633    if cmd.needsGit:
3634        if cmd.gitdir == None:
3635            cmd.gitdir = os.path.abspath(".git")
3636            if not isValidGitDir(cmd.gitdir):
3637                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3638                if os.path.exists(cmd.gitdir):
3639                    cdup = read_pipe("git rev-parse --show-cdup").strip()
3640                    if len(cdup) > 0:
3641                        chdir(cdup);
3642
3643        if not isValidGitDir(cmd.gitdir):
3644            if isValidGitDir(cmd.gitdir + "/.git"):
3645                cmd.gitdir += "/.git"
3646            else:
3647                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3648
3649        os.environ["GIT_DIR"] = cmd.gitdir
3650
3651    if not cmd.run(args):
3652        parser.print_help()
3653        sys.exit(2)
3654
3655
3656if __name__ == '__main__':
3657    main()