git-p4.pyon commit Merge branch 'jk/clone-progress-to-stderr' (f94a84c)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10import sys
  11if sys.hexversion < 0x02040000:
  12    # The limiter is the subprocess module
  13    sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  14    sys.exit(1)
  15import os
  16import optparse
  17import marshal
  18import subprocess
  19import tempfile
  20import time
  21import platform
  22import re
  23import shutil
  24import stat
  25
  26try:
  27    from subprocess import CalledProcessError
  28except ImportError:
  29    # from python2.7:subprocess.py
  30    # Exception classes used by this module.
  31    class CalledProcessError(Exception):
  32        """This exception is raised when a process run by check_call() returns
  33        a non-zero exit status.  The exit status will be stored in the
  34        returncode attribute."""
  35        def __init__(self, returncode, cmd):
  36            self.returncode = returncode
  37            self.cmd = cmd
  38        def __str__(self):
  39            return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
  40
  41verbose = False
  42
  43# Only labels/tags matching this will be imported/exported
  44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  45
  46def p4_build_cmd(cmd):
  47    """Build a suitable p4 command line.
  48
  49    This consolidates building and returning a p4 command line into one
  50    location. It means that hooking into the environment, or other configuration
  51    can be done more easily.
  52    """
  53    real_cmd = ["p4"]
  54
  55    user = gitConfig("git-p4.user")
  56    if len(user) > 0:
  57        real_cmd += ["-u",user]
  58
  59    password = gitConfig("git-p4.password")
  60    if len(password) > 0:
  61        real_cmd += ["-P", password]
  62
  63    port = gitConfig("git-p4.port")
  64    if len(port) > 0:
  65        real_cmd += ["-p", port]
  66
  67    host = gitConfig("git-p4.host")
  68    if len(host) > 0:
  69        real_cmd += ["-H", host]
  70
  71    client = gitConfig("git-p4.client")
  72    if len(client) > 0:
  73        real_cmd += ["-c", client]
  74
  75
  76    if isinstance(cmd,basestring):
  77        real_cmd = ' '.join(real_cmd) + ' ' + cmd
  78    else:
  79        real_cmd += cmd
  80    return real_cmd
  81
  82def chdir(path, is_client_path=False):
  83    """Do chdir to the given path, and set the PWD environment
  84       variable for use by P4.  It does not look at getcwd() output.
  85       Since we're not using the shell, it is necessary to set the
  86       PWD environment variable explicitly.
  87
  88       Normally, expand the path to force it to be absolute.  This
  89       addresses the use of relative path names inside P4 settings,
  90       e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
  91       as given; it looks for .p4config using PWD.
  92
  93       If is_client_path, the path was handed to us directly by p4,
  94       and may be a symbolic link.  Do not call os.getcwd() in this
  95       case, because it will cause p4 to think that PWD is not inside
  96       the client path.
  97       """
  98
  99    os.chdir(path)
 100    if not is_client_path:
 101        path = os.getcwd()
 102    os.environ['PWD'] = path
 103
 104def die(msg):
 105    if verbose:
 106        raise Exception(msg)
 107    else:
 108        sys.stderr.write(msg + "\n")
 109        sys.exit(1)
 110
 111def write_pipe(c, stdin):
 112    if verbose:
 113        sys.stderr.write('Writing pipe: %s\n' % str(c))
 114
 115    expand = isinstance(c,basestring)
 116    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
 117    pipe = p.stdin
 118    val = pipe.write(stdin)
 119    pipe.close()
 120    if p.wait():
 121        die('Command failed: %s' % str(c))
 122
 123    return val
 124
 125def p4_write_pipe(c, stdin):
 126    real_cmd = p4_build_cmd(c)
 127    return write_pipe(real_cmd, stdin)
 128
 129def read_pipe(c, ignore_error=False):
 130    if verbose:
 131        sys.stderr.write('Reading pipe: %s\n' % str(c))
 132
 133    expand = isinstance(c,basestring)
 134    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 135    pipe = p.stdout
 136    val = pipe.read()
 137    if p.wait() and not ignore_error:
 138        die('Command failed: %s' % str(c))
 139
 140    return val
 141
 142def p4_read_pipe(c, ignore_error=False):
 143    real_cmd = p4_build_cmd(c)
 144    return read_pipe(real_cmd, ignore_error)
 145
 146def read_pipe_lines(c):
 147    if verbose:
 148        sys.stderr.write('Reading pipe: %s\n' % str(c))
 149
 150    expand = isinstance(c, basestring)
 151    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 152    pipe = p.stdout
 153    val = pipe.readlines()
 154    if pipe.close() or p.wait():
 155        die('Command failed: %s' % str(c))
 156
 157    return val
 158
 159def p4_read_pipe_lines(c):
 160    """Specifically invoke p4 on the command supplied. """
 161    real_cmd = p4_build_cmd(c)
 162    return read_pipe_lines(real_cmd)
 163
 164def p4_has_command(cmd):
 165    """Ask p4 for help on this command.  If it returns an error, the
 166       command does not exist in this version of p4."""
 167    real_cmd = p4_build_cmd(["help", cmd])
 168    p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 169                                   stderr=subprocess.PIPE)
 170    p.communicate()
 171    return p.returncode == 0
 172
 173def p4_has_move_command():
 174    """See if the move command exists, that it supports -k, and that
 175       it has not been administratively disabled.  The arguments
 176       must be correct, but the filenames do not have to exist.  Use
 177       ones with wildcards so even if they exist, it will fail."""
 178
 179    if not p4_has_command("move"):
 180        return False
 181    cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 182    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 183    (out, err) = p.communicate()
 184    # return code will be 1 in either case
 185    if err.find("Invalid option") >= 0:
 186        return False
 187    if err.find("disabled") >= 0:
 188        return False
 189    # assume it failed because @... was invalid changelist
 190    return True
 191
 192def system(cmd):
 193    expand = isinstance(cmd,basestring)
 194    if verbose:
 195        sys.stderr.write("executing %s\n" % str(cmd))
 196    retcode = subprocess.call(cmd, shell=expand)
 197    if retcode:
 198        raise CalledProcessError(retcode, cmd)
 199
 200def p4_system(cmd):
 201    """Specifically invoke p4 as the system command. """
 202    real_cmd = p4_build_cmd(cmd)
 203    expand = isinstance(real_cmd, basestring)
 204    retcode = subprocess.call(real_cmd, shell=expand)
 205    if retcode:
 206        raise CalledProcessError(retcode, real_cmd)
 207
 208_p4_version_string = None
 209def p4_version_string():
 210    """Read the version string, showing just the last line, which
 211       hopefully is the interesting version bit.
 212
 213       $ p4 -V
 214       Perforce - The Fast Software Configuration Management System.
 215       Copyright 1995-2011 Perforce Software.  All rights reserved.
 216       Rev. P4/NTX86/2011.1/393975 (2011/12/16).
 217    """
 218    global _p4_version_string
 219    if not _p4_version_string:
 220        a = p4_read_pipe_lines(["-V"])
 221        _p4_version_string = a[-1].rstrip()
 222    return _p4_version_string
 223
 224def p4_integrate(src, dest):
 225    p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 226
 227def p4_sync(f, *options):
 228    p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 229
 230def p4_add(f):
 231    # forcibly add file names with wildcards
 232    if wildcard_present(f):
 233        p4_system(["add", "-f", f])
 234    else:
 235        p4_system(["add", f])
 236
 237def p4_delete(f):
 238    p4_system(["delete", wildcard_encode(f)])
 239
 240def p4_edit(f):
 241    p4_system(["edit", wildcard_encode(f)])
 242
 243def p4_revert(f):
 244    p4_system(["revert", wildcard_encode(f)])
 245
 246def p4_reopen(type, f):
 247    p4_system(["reopen", "-t", type, wildcard_encode(f)])
 248
 249def p4_move(src, dest):
 250    p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 251
 252def p4_describe(change):
 253    """Make sure it returns a valid result by checking for
 254       the presence of field "time".  Return a dict of the
 255       results."""
 256
 257    ds = p4CmdList(["describe", "-s", str(change)])
 258    if len(ds) != 1:
 259        die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 260
 261    d = ds[0]
 262
 263    if "p4ExitCode" in d:
 264        die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 265                                                      str(d)))
 266    if "code" in d:
 267        if d["code"] == "error":
 268            die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 269
 270    if "time" not in d:
 271        die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 272
 273    return d
 274
 275#
 276# Canonicalize the p4 type and return a tuple of the
 277# base type, plus any modifiers.  See "p4 help filetypes"
 278# for a list and explanation.
 279#
 280def split_p4_type(p4type):
 281
 282    p4_filetypes_historical = {
 283        "ctempobj": "binary+Sw",
 284        "ctext": "text+C",
 285        "cxtext": "text+Cx",
 286        "ktext": "text+k",
 287        "kxtext": "text+kx",
 288        "ltext": "text+F",
 289        "tempobj": "binary+FSw",
 290        "ubinary": "binary+F",
 291        "uresource": "resource+F",
 292        "uxbinary": "binary+Fx",
 293        "xbinary": "binary+x",
 294        "xltext": "text+Fx",
 295        "xtempobj": "binary+Swx",
 296        "xtext": "text+x",
 297        "xunicode": "unicode+x",
 298        "xutf16": "utf16+x",
 299    }
 300    if p4type in p4_filetypes_historical:
 301        p4type = p4_filetypes_historical[p4type]
 302    mods = ""
 303    s = p4type.split("+")
 304    base = s[0]
 305    mods = ""
 306    if len(s) > 1:
 307        mods = s[1]
 308    return (base, mods)
 309
 310#
 311# return the raw p4 type of a file (text, text+ko, etc)
 312#
 313def p4_type(file):
 314    results = p4CmdList(["fstat", "-T", "headType", file])
 315    return results[0]['headType']
 316
 317#
 318# Given a type base and modifier, return a regexp matching
 319# the keywords that can be expanded in the file
 320#
 321def p4_keywords_regexp_for_type(base, type_mods):
 322    if base in ("text", "unicode", "binary"):
 323        kwords = None
 324        if "ko" in type_mods:
 325            kwords = 'Id|Header'
 326        elif "k" in type_mods:
 327            kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 328        else:
 329            return None
 330        pattern = r"""
 331            \$              # Starts with a dollar, followed by...
 332            (%s)            # one of the keywords, followed by...
 333            (:[^$\n]+)?     # possibly an old expansion, followed by...
 334            \$              # another dollar
 335            """ % kwords
 336        return pattern
 337    else:
 338        return None
 339
 340#
 341# Given a file, return a regexp matching the possible
 342# RCS keywords that will be expanded, or None for files
 343# with kw expansion turned off.
 344#
 345def p4_keywords_regexp_for_file(file):
 346    if not os.path.exists(file):
 347        return None
 348    else:
 349        (type_base, type_mods) = split_p4_type(p4_type(file))
 350        return p4_keywords_regexp_for_type(type_base, type_mods)
 351
 352def setP4ExecBit(file, mode):
 353    # Reopens an already open file and changes the execute bit to match
 354    # the execute bit setting in the passed in mode.
 355
 356    p4Type = "+x"
 357
 358    if not isModeExec(mode):
 359        p4Type = getP4OpenedType(file)
 360        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 361        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 362        if p4Type[-1] == "+":
 363            p4Type = p4Type[0:-1]
 364
 365    p4_reopen(p4Type, file)
 366
 367def getP4OpenedType(file):
 368    # Returns the perforce file type for the given file.
 369
 370    result = p4_read_pipe(["opened", wildcard_encode(file)])
 371    match = re.match(".*\((.+)\)\r?$", result)
 372    if match:
 373        return match.group(1)
 374    else:
 375        die("Could not determine file type for %s (result: '%s')" % (file, result))
 376
 377# Return the set of all p4 labels
 378def getP4Labels(depotPaths):
 379    labels = set()
 380    if isinstance(depotPaths,basestring):
 381        depotPaths = [depotPaths]
 382
 383    for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 384        label = l['label']
 385        labels.add(label)
 386
 387    return labels
 388
 389# Return the set of all git tags
 390def getGitTags():
 391    gitTags = set()
 392    for line in read_pipe_lines(["git", "tag"]):
 393        tag = line.strip()
 394        gitTags.add(tag)
 395    return gitTags
 396
 397def diffTreePattern():
 398    # This is a simple generator for the diff tree regex pattern. This could be
 399    # a class variable if this and parseDiffTreeEntry were a part of a class.
 400    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 401    while True:
 402        yield pattern
 403
 404def parseDiffTreeEntry(entry):
 405    """Parses a single diff tree entry into its component elements.
 406
 407    See git-diff-tree(1) manpage for details about the format of the diff
 408    output. This method returns a dictionary with the following elements:
 409
 410    src_mode - The mode of the source file
 411    dst_mode - The mode of the destination file
 412    src_sha1 - The sha1 for the source file
 413    dst_sha1 - The sha1 fr the destination file
 414    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 415    status_score - The score for the status (applicable for 'C' and 'R'
 416                   statuses). This is None if there is no score.
 417    src - The path for the source file.
 418    dst - The path for the destination file. This is only present for
 419          copy or renames. If it is not present, this is None.
 420
 421    If the pattern is not matched, None is returned."""
 422
 423    match = diffTreePattern().next().match(entry)
 424    if match:
 425        return {
 426            'src_mode': match.group(1),
 427            'dst_mode': match.group(2),
 428            'src_sha1': match.group(3),
 429            'dst_sha1': match.group(4),
 430            'status': match.group(5),
 431            'status_score': match.group(6),
 432            'src': match.group(7),
 433            'dst': match.group(10)
 434        }
 435    return None
 436
 437def isModeExec(mode):
 438    # Returns True if the given git mode represents an executable file,
 439    # otherwise False.
 440    return mode[-3:] == "755"
 441
 442def isModeExecChanged(src_mode, dst_mode):
 443    return isModeExec(src_mode) != isModeExec(dst_mode)
 444
 445def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 446
 447    if isinstance(cmd,basestring):
 448        cmd = "-G " + cmd
 449        expand = True
 450    else:
 451        cmd = ["-G"] + cmd
 452        expand = False
 453
 454    cmd = p4_build_cmd(cmd)
 455    if verbose:
 456        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 457
 458    # Use a temporary file to avoid deadlocks without
 459    # subprocess.communicate(), which would put another copy
 460    # of stdout into memory.
 461    stdin_file = None
 462    if stdin is not None:
 463        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 464        if isinstance(stdin,basestring):
 465            stdin_file.write(stdin)
 466        else:
 467            for i in stdin:
 468                stdin_file.write(i + '\n')
 469        stdin_file.flush()
 470        stdin_file.seek(0)
 471
 472    p4 = subprocess.Popen(cmd,
 473                          shell=expand,
 474                          stdin=stdin_file,
 475                          stdout=subprocess.PIPE)
 476
 477    result = []
 478    try:
 479        while True:
 480            entry = marshal.load(p4.stdout)
 481            if cb is not None:
 482                cb(entry)
 483            else:
 484                result.append(entry)
 485    except EOFError:
 486        pass
 487    exitCode = p4.wait()
 488    if exitCode != 0:
 489        entry = {}
 490        entry["p4ExitCode"] = exitCode
 491        result.append(entry)
 492
 493    return result
 494
 495def p4Cmd(cmd):
 496    list = p4CmdList(cmd)
 497    result = {}
 498    for entry in list:
 499        result.update(entry)
 500    return result;
 501
 502def p4Where(depotPath):
 503    if not depotPath.endswith("/"):
 504        depotPath += "/"
 505    depotPath = depotPath + "..."
 506    outputList = p4CmdList(["where", depotPath])
 507    output = None
 508    for entry in outputList:
 509        if "depotFile" in entry:
 510            if entry["depotFile"] == depotPath:
 511                output = entry
 512                break
 513        elif "data" in entry:
 514            data = entry.get("data")
 515            space = data.find(" ")
 516            if data[:space] == depotPath:
 517                output = entry
 518                break
 519    if output == None:
 520        return ""
 521    if output["code"] == "error":
 522        return ""
 523    clientPath = ""
 524    if "path" in output:
 525        clientPath = output.get("path")
 526    elif "data" in output:
 527        data = output.get("data")
 528        lastSpace = data.rfind(" ")
 529        clientPath = data[lastSpace + 1:]
 530
 531    if clientPath.endswith("..."):
 532        clientPath = clientPath[:-3]
 533    return clientPath
 534
 535def currentGitBranch():
 536    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 537
 538def isValidGitDir(path):
 539    if (os.path.exists(path + "/HEAD")
 540        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 541        return True;
 542    return False
 543
 544def parseRevision(ref):
 545    return read_pipe("git rev-parse %s" % ref).strip()
 546
 547def branchExists(ref):
 548    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 549                     ignore_error=True)
 550    return len(rev) > 0
 551
 552def extractLogMessageFromGitCommit(commit):
 553    logMessage = ""
 554
 555    ## fixme: title is first line of commit, not 1st paragraph.
 556    foundTitle = False
 557    for log in read_pipe_lines("git cat-file commit %s" % commit):
 558       if not foundTitle:
 559           if len(log) == 1:
 560               foundTitle = True
 561           continue
 562
 563       logMessage += log
 564    return logMessage
 565
 566def extractSettingsGitLog(log):
 567    values = {}
 568    for line in log.split("\n"):
 569        line = line.strip()
 570        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 571        if not m:
 572            continue
 573
 574        assignments = m.group(1).split (':')
 575        for a in assignments:
 576            vals = a.split ('=')
 577            key = vals[0].strip()
 578            val = ('='.join (vals[1:])).strip()
 579            if val.endswith ('\"') and val.startswith('"'):
 580                val = val[1:-1]
 581
 582            values[key] = val
 583
 584    paths = values.get("depot-paths")
 585    if not paths:
 586        paths = values.get("depot-path")
 587    if paths:
 588        values['depot-paths'] = paths.split(',')
 589    return values
 590
 591def gitBranchExists(branch):
 592    proc = subprocess.Popen(["git", "rev-parse", branch],
 593                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 594    return proc.wait() == 0;
 595
 596_gitConfig = {}
 597
 598def gitConfig(key):
 599    if not _gitConfig.has_key(key):
 600        cmd = [ "git", "config", key ]
 601        s = read_pipe(cmd, ignore_error=True)
 602        _gitConfig[key] = s.strip()
 603    return _gitConfig[key]
 604
 605def gitConfigBool(key):
 606    """Return a bool, using git config --bool.  It is True only if the
 607       variable is set to true, and False if set to false or not present
 608       in the config."""
 609
 610    if not _gitConfig.has_key(key):
 611        cmd = [ "git", "config", "--bool", key ]
 612        s = read_pipe(cmd, ignore_error=True)
 613        v = s.strip()
 614        _gitConfig[key] = v == "true"
 615    return _gitConfig[key]
 616
 617def gitConfigList(key):
 618    if not _gitConfig.has_key(key):
 619        s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
 620        _gitConfig[key] = s.strip().split(os.linesep)
 621    return _gitConfig[key]
 622
 623def p4BranchesInGit(branchesAreInRemotes=True):
 624    """Find all the branches whose names start with "p4/", looking
 625       in remotes or heads as specified by the argument.  Return
 626       a dictionary of { branch: revision } for each one found.
 627       The branch names are the short names, without any
 628       "p4/" prefix."""
 629
 630    branches = {}
 631
 632    cmdline = "git rev-parse --symbolic "
 633    if branchesAreInRemotes:
 634        cmdline += "--remotes"
 635    else:
 636        cmdline += "--branches"
 637
 638    for line in read_pipe_lines(cmdline):
 639        line = line.strip()
 640
 641        # only import to p4/
 642        if not line.startswith('p4/'):
 643            continue
 644        # special symbolic ref to p4/master
 645        if line == "p4/HEAD":
 646            continue
 647
 648        # strip off p4/ prefix
 649        branch = line[len("p4/"):]
 650
 651        branches[branch] = parseRevision(line)
 652
 653    return branches
 654
 655def branch_exists(branch):
 656    """Make sure that the given ref name really exists."""
 657
 658    cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 659    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 660    out, _ = p.communicate()
 661    if p.returncode:
 662        return False
 663    # expect exactly one line of output: the branch name
 664    return out.rstrip() == branch
 665
 666def findUpstreamBranchPoint(head = "HEAD"):
 667    branches = p4BranchesInGit()
 668    # map from depot-path to branch name
 669    branchByDepotPath = {}
 670    for branch in branches.keys():
 671        tip = branches[branch]
 672        log = extractLogMessageFromGitCommit(tip)
 673        settings = extractSettingsGitLog(log)
 674        if settings.has_key("depot-paths"):
 675            paths = ",".join(settings["depot-paths"])
 676            branchByDepotPath[paths] = "remotes/p4/" + branch
 677
 678    settings = None
 679    parent = 0
 680    while parent < 65535:
 681        commit = head + "~%s" % parent
 682        log = extractLogMessageFromGitCommit(commit)
 683        settings = extractSettingsGitLog(log)
 684        if settings.has_key("depot-paths"):
 685            paths = ",".join(settings["depot-paths"])
 686            if branchByDepotPath.has_key(paths):
 687                return [branchByDepotPath[paths], settings]
 688
 689        parent = parent + 1
 690
 691    return ["", settings]
 692
 693def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 694    if not silent:
 695        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 696               % localRefPrefix)
 697
 698    originPrefix = "origin/p4/"
 699
 700    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 701        line = line.strip()
 702        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 703            continue
 704
 705        headName = line[len(originPrefix):]
 706        remoteHead = localRefPrefix + headName
 707        originHead = line
 708
 709        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 710        if (not original.has_key('depot-paths')
 711            or not original.has_key('change')):
 712            continue
 713
 714        update = False
 715        if not gitBranchExists(remoteHead):
 716            if verbose:
 717                print "creating %s" % remoteHead
 718            update = True
 719        else:
 720            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 721            if settings.has_key('change') > 0:
 722                if settings['depot-paths'] == original['depot-paths']:
 723                    originP4Change = int(original['change'])
 724                    p4Change = int(settings['change'])
 725                    if originP4Change > p4Change:
 726                        print ("%s (%s) is newer than %s (%s). "
 727                               "Updating p4 branch from origin."
 728                               % (originHead, originP4Change,
 729                                  remoteHead, p4Change))
 730                        update = True
 731                else:
 732                    print ("Ignoring: %s was imported from %s while "
 733                           "%s was imported from %s"
 734                           % (originHead, ','.join(original['depot-paths']),
 735                              remoteHead, ','.join(settings['depot-paths'])))
 736
 737        if update:
 738            system("git update-ref %s %s" % (remoteHead, originHead))
 739
 740def originP4BranchesExist():
 741        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 742
 743def p4ChangesForPaths(depotPaths, changeRange):
 744    assert depotPaths
 745    cmd = ['changes']
 746    for p in depotPaths:
 747        cmd += ["%s...%s" % (p, changeRange)]
 748    output = p4_read_pipe_lines(cmd)
 749
 750    changes = {}
 751    for line in output:
 752        changeNum = int(line.split(" ")[1])
 753        changes[changeNum] = True
 754
 755    changelist = changes.keys()
 756    changelist.sort()
 757    return changelist
 758
 759def p4PathStartsWith(path, prefix):
 760    # This method tries to remedy a potential mixed-case issue:
 761    #
 762    # If UserA adds  //depot/DirA/file1
 763    # and UserB adds //depot/dira/file2
 764    #
 765    # we may or may not have a problem. If you have core.ignorecase=true,
 766    # we treat DirA and dira as the same directory
 767    if gitConfigBool("core.ignorecase"):
 768        return path.lower().startswith(prefix.lower())
 769    return path.startswith(prefix)
 770
 771def getClientSpec():
 772    """Look at the p4 client spec, create a View() object that contains
 773       all the mappings, and return it."""
 774
 775    specList = p4CmdList("client -o")
 776    if len(specList) != 1:
 777        die('Output from "client -o" is %d lines, expecting 1' %
 778            len(specList))
 779
 780    # dictionary of all client parameters
 781    entry = specList[0]
 782
 783    # the //client/ name
 784    client_name = entry["Client"]
 785
 786    # just the keys that start with "View"
 787    view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 788
 789    # hold this new View
 790    view = View(client_name)
 791
 792    # append the lines, in order, to the view
 793    for view_num in range(len(view_keys)):
 794        k = "View%d" % view_num
 795        if k not in view_keys:
 796            die("Expected view key %s missing" % k)
 797        view.append(entry[k])
 798
 799    return view
 800
 801def getClientRoot():
 802    """Grab the client directory."""
 803
 804    output = p4CmdList("client -o")
 805    if len(output) != 1:
 806        die('Output from "client -o" is %d lines, expecting 1' % len(output))
 807
 808    entry = output[0]
 809    if "Root" not in entry:
 810        die('Client has no "Root"')
 811
 812    return entry["Root"]
 813
 814#
 815# P4 wildcards are not allowed in filenames.  P4 complains
 816# if you simply add them, but you can force it with "-f", in
 817# which case it translates them into %xx encoding internally.
 818#
 819def wildcard_decode(path):
 820    # Search for and fix just these four characters.  Do % last so
 821    # that fixing it does not inadvertently create new %-escapes.
 822    # Cannot have * in a filename in windows; untested as to
 823    # what p4 would do in such a case.
 824    if not platform.system() == "Windows":
 825        path = path.replace("%2A", "*")
 826    path = path.replace("%23", "#") \
 827               .replace("%40", "@") \
 828               .replace("%25", "%")
 829    return path
 830
 831def wildcard_encode(path):
 832    # do % first to avoid double-encoding the %s introduced here
 833    path = path.replace("%", "%25") \
 834               .replace("*", "%2A") \
 835               .replace("#", "%23") \
 836               .replace("@", "%40")
 837    return path
 838
 839def wildcard_present(path):
 840    m = re.search("[*#@%]", path)
 841    return m is not None
 842
 843class Command:
 844    def __init__(self):
 845        self.usage = "usage: %prog [options]"
 846        self.needsGit = True
 847        self.verbose = False
 848
 849class P4UserMap:
 850    def __init__(self):
 851        self.userMapFromPerforceServer = False
 852        self.myP4UserId = None
 853
 854    def p4UserId(self):
 855        if self.myP4UserId:
 856            return self.myP4UserId
 857
 858        results = p4CmdList("user -o")
 859        for r in results:
 860            if r.has_key('User'):
 861                self.myP4UserId = r['User']
 862                return r['User']
 863        die("Could not find your p4 user id")
 864
 865    def p4UserIsMe(self, p4User):
 866        # return True if the given p4 user is actually me
 867        me = self.p4UserId()
 868        if not p4User or p4User != me:
 869            return False
 870        else:
 871            return True
 872
 873    def getUserCacheFilename(self):
 874        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 875        return home + "/.gitp4-usercache.txt"
 876
 877    def getUserMapFromPerforceServer(self):
 878        if self.userMapFromPerforceServer:
 879            return
 880        self.users = {}
 881        self.emails = {}
 882
 883        for output in p4CmdList("users"):
 884            if not output.has_key("User"):
 885                continue
 886            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 887            self.emails[output["Email"]] = output["User"]
 888
 889
 890        s = ''
 891        for (key, val) in self.users.items():
 892            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 893
 894        open(self.getUserCacheFilename(), "wb").write(s)
 895        self.userMapFromPerforceServer = True
 896
 897    def loadUserMapFromCache(self):
 898        self.users = {}
 899        self.userMapFromPerforceServer = False
 900        try:
 901            cache = open(self.getUserCacheFilename(), "rb")
 902            lines = cache.readlines()
 903            cache.close()
 904            for line in lines:
 905                entry = line.strip().split("\t")
 906                self.users[entry[0]] = entry[1]
 907        except IOError:
 908            self.getUserMapFromPerforceServer()
 909
 910class P4Debug(Command):
 911    def __init__(self):
 912        Command.__init__(self)
 913        self.options = []
 914        self.description = "A tool to debug the output of p4 -G."
 915        self.needsGit = False
 916
 917    def run(self, args):
 918        j = 0
 919        for output in p4CmdList(args):
 920            print 'Element: %d' % j
 921            j += 1
 922            print output
 923        return True
 924
 925class P4RollBack(Command):
 926    def __init__(self):
 927        Command.__init__(self)
 928        self.options = [
 929            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 930        ]
 931        self.description = "A tool to debug the multi-branch import. Don't use :)"
 932        self.rollbackLocalBranches = False
 933
 934    def run(self, args):
 935        if len(args) != 1:
 936            return False
 937        maxChange = int(args[0])
 938
 939        if "p4ExitCode" in p4Cmd("changes -m 1"):
 940            die("Problems executing p4");
 941
 942        if self.rollbackLocalBranches:
 943            refPrefix = "refs/heads/"
 944            lines = read_pipe_lines("git rev-parse --symbolic --branches")
 945        else:
 946            refPrefix = "refs/remotes/"
 947            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 948
 949        for line in lines:
 950            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 951                line = line.strip()
 952                ref = refPrefix + line
 953                log = extractLogMessageFromGitCommit(ref)
 954                settings = extractSettingsGitLog(log)
 955
 956                depotPaths = settings['depot-paths']
 957                change = settings['change']
 958
 959                changed = False
 960
 961                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 962                                                           for p in depotPaths]))) == 0:
 963                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 964                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 965                    continue
 966
 967                while change and int(change) > maxChange:
 968                    changed = True
 969                    if self.verbose:
 970                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 971                    system("git update-ref %s \"%s^\"" % (ref, ref))
 972                    log = extractLogMessageFromGitCommit(ref)
 973                    settings =  extractSettingsGitLog(log)
 974
 975
 976                    depotPaths = settings['depot-paths']
 977                    change = settings['change']
 978
 979                if changed:
 980                    print "%s rewound to %s" % (ref, change)
 981
 982        return True
 983
 984class P4Submit(Command, P4UserMap):
 985
 986    conflict_behavior_choices = ("ask", "skip", "quit")
 987
 988    def __init__(self):
 989        Command.__init__(self)
 990        P4UserMap.__init__(self)
 991        self.options = [
 992                optparse.make_option("--origin", dest="origin"),
 993                optparse.make_option("-M", dest="detectRenames", action="store_true"),
 994                # preserve the user, requires relevant p4 permissions
 995                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 996                optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
 997                optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
 998                optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
 999                optparse.make_option("--conflict", dest="conflict_behavior",
1000                                     choices=self.conflict_behavior_choices),
1001                optparse.make_option("--branch", dest="branch"),
1002        ]
1003        self.description = "Submit changes from git to the perforce depot."
1004        self.usage += " [name of git branch to submit into perforce depot]"
1005        self.origin = ""
1006        self.detectRenames = False
1007        self.preserveUser = gitConfigBool("git-p4.preserveUser")
1008        self.dry_run = False
1009        self.prepare_p4_only = False
1010        self.conflict_behavior = None
1011        self.isWindows = (platform.system() == "Windows")
1012        self.exportLabels = False
1013        self.p4HasMoveCommand = p4_has_move_command()
1014        self.branch = None
1015
1016    def check(self):
1017        if len(p4CmdList("opened ...")) > 0:
1018            die("You have files opened with perforce! Close them before starting the sync.")
1019
1020    def separate_jobs_from_description(self, message):
1021        """Extract and return a possible Jobs field in the commit
1022           message.  It goes into a separate section in the p4 change
1023           specification.
1024
1025           A jobs line starts with "Jobs:" and looks like a new field
1026           in a form.  Values are white-space separated on the same
1027           line or on following lines that start with a tab.
1028
1029           This does not parse and extract the full git commit message
1030           like a p4 form.  It just sees the Jobs: line as a marker
1031           to pass everything from then on directly into the p4 form,
1032           but outside the description section.
1033
1034           Return a tuple (stripped log message, jobs string)."""
1035
1036        m = re.search(r'^Jobs:', message, re.MULTILINE)
1037        if m is None:
1038            return (message, None)
1039
1040        jobtext = message[m.start():]
1041        stripped_message = message[:m.start()].rstrip()
1042        return (stripped_message, jobtext)
1043
1044    def prepareLogMessage(self, template, message, jobs):
1045        """Edits the template returned from "p4 change -o" to insert
1046           the message in the Description field, and the jobs text in
1047           the Jobs field."""
1048        result = ""
1049
1050        inDescriptionSection = False
1051
1052        for line in template.split("\n"):
1053            if line.startswith("#"):
1054                result += line + "\n"
1055                continue
1056
1057            if inDescriptionSection:
1058                if line.startswith("Files:") or line.startswith("Jobs:"):
1059                    inDescriptionSection = False
1060                    # insert Jobs section
1061                    if jobs:
1062                        result += jobs + "\n"
1063                else:
1064                    continue
1065            else:
1066                if line.startswith("Description:"):
1067                    inDescriptionSection = True
1068                    line += "\n"
1069                    for messageLine in message.split("\n"):
1070                        line += "\t" + messageLine + "\n"
1071
1072            result += line + "\n"
1073
1074        return result
1075
1076    def patchRCSKeywords(self, file, pattern):
1077        # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1078        (handle, outFileName) = tempfile.mkstemp(dir='.')
1079        try:
1080            outFile = os.fdopen(handle, "w+")
1081            inFile = open(file, "r")
1082            regexp = re.compile(pattern, re.VERBOSE)
1083            for line in inFile.readlines():
1084                line = regexp.sub(r'$\1$', line)
1085                outFile.write(line)
1086            inFile.close()
1087            outFile.close()
1088            # Forcibly overwrite the original file
1089            os.unlink(file)
1090            shutil.move(outFileName, file)
1091        except:
1092            # cleanup our temporary file
1093            os.unlink(outFileName)
1094            print "Failed to strip RCS keywords in %s" % file
1095            raise
1096
1097        print "Patched up RCS keywords in %s" % file
1098
1099    def p4UserForCommit(self,id):
1100        # Return the tuple (perforce user,git email) for a given git commit id
1101        self.getUserMapFromPerforceServer()
1102        gitEmail = read_pipe(["git", "log", "--max-count=1",
1103                              "--format=%ae", id])
1104        gitEmail = gitEmail.strip()
1105        if not self.emails.has_key(gitEmail):
1106            return (None,gitEmail)
1107        else:
1108            return (self.emails[gitEmail],gitEmail)
1109
1110    def checkValidP4Users(self,commits):
1111        # check if any git authors cannot be mapped to p4 users
1112        for id in commits:
1113            (user,email) = self.p4UserForCommit(id)
1114            if not user:
1115                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1116                if gitConfigBool("git-p4.allowMissingP4Users"):
1117                    print "%s" % msg
1118                else:
1119                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1120
1121    def lastP4Changelist(self):
1122        # Get back the last changelist number submitted in this client spec. This
1123        # then gets used to patch up the username in the change. If the same
1124        # client spec is being used by multiple processes then this might go
1125        # wrong.
1126        results = p4CmdList("client -o")        # find the current client
1127        client = None
1128        for r in results:
1129            if r.has_key('Client'):
1130                client = r['Client']
1131                break
1132        if not client:
1133            die("could not get client spec")
1134        results = p4CmdList(["changes", "-c", client, "-m", "1"])
1135        for r in results:
1136            if r.has_key('change'):
1137                return r['change']
1138        die("Could not get changelist number for last submit - cannot patch up user details")
1139
1140    def modifyChangelistUser(self, changelist, newUser):
1141        # fixup the user field of a changelist after it has been submitted.
1142        changes = p4CmdList("change -o %s" % changelist)
1143        if len(changes) != 1:
1144            die("Bad output from p4 change modifying %s to user %s" %
1145                (changelist, newUser))
1146
1147        c = changes[0]
1148        if c['User'] == newUser: return   # nothing to do
1149        c['User'] = newUser
1150        input = marshal.dumps(c)
1151
1152        result = p4CmdList("change -f -i", stdin=input)
1153        for r in result:
1154            if r.has_key('code'):
1155                if r['code'] == 'error':
1156                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1157            if r.has_key('data'):
1158                print("Updated user field for changelist %s to %s" % (changelist, newUser))
1159                return
1160        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1161
1162    def canChangeChangelists(self):
1163        # check to see if we have p4 admin or super-user permissions, either of
1164        # which are required to modify changelists.
1165        results = p4CmdList(["protects", self.depotPath])
1166        for r in results:
1167            if r.has_key('perm'):
1168                if r['perm'] == 'admin':
1169                    return 1
1170                if r['perm'] == 'super':
1171                    return 1
1172        return 0
1173
1174    def prepareSubmitTemplate(self):
1175        """Run "p4 change -o" to grab a change specification template.
1176           This does not use "p4 -G", as it is nice to keep the submission
1177           template in original order, since a human might edit it.
1178
1179           Remove lines in the Files section that show changes to files
1180           outside the depot path we're committing into."""
1181
1182        template = ""
1183        inFilesSection = False
1184        for line in p4_read_pipe_lines(['change', '-o']):
1185            if line.endswith("\r\n"):
1186                line = line[:-2] + "\n"
1187            if inFilesSection:
1188                if line.startswith("\t"):
1189                    # path starts and ends with a tab
1190                    path = line[1:]
1191                    lastTab = path.rfind("\t")
1192                    if lastTab != -1:
1193                        path = path[:lastTab]
1194                        if not p4PathStartsWith(path, self.depotPath):
1195                            continue
1196                else:
1197                    inFilesSection = False
1198            else:
1199                if line.startswith("Files:"):
1200                    inFilesSection = True
1201
1202            template += line
1203
1204        return template
1205
1206    def edit_template(self, template_file):
1207        """Invoke the editor to let the user change the submission
1208           message.  Return true if okay to continue with the submit."""
1209
1210        # if configured to skip the editing part, just submit
1211        if gitConfigBool("git-p4.skipSubmitEdit"):
1212            return True
1213
1214        # look at the modification time, to check later if the user saved
1215        # the file
1216        mtime = os.stat(template_file).st_mtime
1217
1218        # invoke the editor
1219        if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1220            editor = os.environ.get("P4EDITOR")
1221        else:
1222            editor = read_pipe("git var GIT_EDITOR").strip()
1223        system(editor + " " + template_file)
1224
1225        # If the file was not saved, prompt to see if this patch should
1226        # be skipped.  But skip this verification step if configured so.
1227        if gitConfigBool("git-p4.skipSubmitEditCheck"):
1228            return True
1229
1230        # modification time updated means user saved the file
1231        if os.stat(template_file).st_mtime > mtime:
1232            return True
1233
1234        while True:
1235            response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1236            if response == 'y':
1237                return True
1238            if response == 'n':
1239                return False
1240
1241    def applyCommit(self, id):
1242        """Apply one commit, return True if it succeeded."""
1243
1244        print "Applying", read_pipe(["git", "show", "-s",
1245                                     "--format=format:%h %s", id])
1246
1247        (p4User, gitEmail) = self.p4UserForCommit(id)
1248
1249        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1250        filesToAdd = set()
1251        filesToDelete = set()
1252        editedFiles = set()
1253        pureRenameCopy = set()
1254        filesToChangeExecBit = {}
1255
1256        for line in diff:
1257            diff = parseDiffTreeEntry(line)
1258            modifier = diff['status']
1259            path = diff['src']
1260            if modifier == "M":
1261                p4_edit(path)
1262                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1263                    filesToChangeExecBit[path] = diff['dst_mode']
1264                editedFiles.add(path)
1265            elif modifier == "A":
1266                filesToAdd.add(path)
1267                filesToChangeExecBit[path] = diff['dst_mode']
1268                if path in filesToDelete:
1269                    filesToDelete.remove(path)
1270            elif modifier == "D":
1271                filesToDelete.add(path)
1272                if path in filesToAdd:
1273                    filesToAdd.remove(path)
1274            elif modifier == "C":
1275                src, dest = diff['src'], diff['dst']
1276                p4_integrate(src, dest)
1277                pureRenameCopy.add(dest)
1278                if diff['src_sha1'] != diff['dst_sha1']:
1279                    p4_edit(dest)
1280                    pureRenameCopy.discard(dest)
1281                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1282                    p4_edit(dest)
1283                    pureRenameCopy.discard(dest)
1284                    filesToChangeExecBit[dest] = diff['dst_mode']
1285                if self.isWindows:
1286                    # turn off read-only attribute
1287                    os.chmod(dest, stat.S_IWRITE)
1288                os.unlink(dest)
1289                editedFiles.add(dest)
1290            elif modifier == "R":
1291                src, dest = diff['src'], diff['dst']
1292                if self.p4HasMoveCommand:
1293                    p4_edit(src)        # src must be open before move
1294                    p4_move(src, dest)  # opens for (move/delete, move/add)
1295                else:
1296                    p4_integrate(src, dest)
1297                    if diff['src_sha1'] != diff['dst_sha1']:
1298                        p4_edit(dest)
1299                    else:
1300                        pureRenameCopy.add(dest)
1301                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1302                    if not self.p4HasMoveCommand:
1303                        p4_edit(dest)   # with move: already open, writable
1304                    filesToChangeExecBit[dest] = diff['dst_mode']
1305                if not self.p4HasMoveCommand:
1306                    if self.isWindows:
1307                        os.chmod(dest, stat.S_IWRITE)
1308                    os.unlink(dest)
1309                    filesToDelete.add(src)
1310                editedFiles.add(dest)
1311            else:
1312                die("unknown modifier %s for %s" % (modifier, path))
1313
1314        diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
1315        patchcmd = diffcmd + " | git apply "
1316        tryPatchCmd = patchcmd + "--check -"
1317        applyPatchCmd = patchcmd + "--check --apply -"
1318        patch_succeeded = True
1319
1320        if os.system(tryPatchCmd) != 0:
1321            fixed_rcs_keywords = False
1322            patch_succeeded = False
1323            print "Unfortunately applying the change failed!"
1324
1325            # Patch failed, maybe it's just RCS keyword woes. Look through
1326            # the patch to see if that's possible.
1327            if gitConfigBool("git-p4.attemptRCSCleanup"):
1328                file = None
1329                pattern = None
1330                kwfiles = {}
1331                for file in editedFiles | filesToDelete:
1332                    # did this file's delta contain RCS keywords?
1333                    pattern = p4_keywords_regexp_for_file(file)
1334
1335                    if pattern:
1336                        # this file is a possibility...look for RCS keywords.
1337                        regexp = re.compile(pattern, re.VERBOSE)
1338                        for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1339                            if regexp.search(line):
1340                                if verbose:
1341                                    print "got keyword match on %s in %s in %s" % (pattern, line, file)
1342                                kwfiles[file] = pattern
1343                                break
1344
1345                for file in kwfiles:
1346                    if verbose:
1347                        print "zapping %s with %s" % (line,pattern)
1348                    # File is being deleted, so not open in p4.  Must
1349                    # disable the read-only bit on windows.
1350                    if self.isWindows and file not in editedFiles:
1351                        os.chmod(file, stat.S_IWRITE)
1352                    self.patchRCSKeywords(file, kwfiles[file])
1353                    fixed_rcs_keywords = True
1354
1355            if fixed_rcs_keywords:
1356                print "Retrying the patch with RCS keywords cleaned up"
1357                if os.system(tryPatchCmd) == 0:
1358                    patch_succeeded = True
1359
1360        if not patch_succeeded:
1361            for f in editedFiles:
1362                p4_revert(f)
1363            return False
1364
1365        #
1366        # Apply the patch for real, and do add/delete/+x handling.
1367        #
1368        system(applyPatchCmd)
1369
1370        for f in filesToAdd:
1371            p4_add(f)
1372        for f in filesToDelete:
1373            p4_revert(f)
1374            p4_delete(f)
1375
1376        # Set/clear executable bits
1377        for f in filesToChangeExecBit.keys():
1378            mode = filesToChangeExecBit[f]
1379            setP4ExecBit(f, mode)
1380
1381        #
1382        # Build p4 change description, starting with the contents
1383        # of the git commit message.
1384        #
1385        logMessage = extractLogMessageFromGitCommit(id)
1386        logMessage = logMessage.strip()
1387        (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1388
1389        template = self.prepareSubmitTemplate()
1390        submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1391
1392        if self.preserveUser:
1393           submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1394
1395        if self.checkAuthorship and not self.p4UserIsMe(p4User):
1396            submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1397            submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1398            submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1399
1400        separatorLine = "######## everything below this line is just the diff #######\n"
1401
1402        # diff
1403        if os.environ.has_key("P4DIFF"):
1404            del(os.environ["P4DIFF"])
1405        diff = ""
1406        for editedFile in editedFiles:
1407            diff += p4_read_pipe(['diff', '-du',
1408                                  wildcard_encode(editedFile)])
1409
1410        # new file diff
1411        newdiff = ""
1412        for newFile in filesToAdd:
1413            newdiff += "==== new file ====\n"
1414            newdiff += "--- /dev/null\n"
1415            newdiff += "+++ %s\n" % newFile
1416            f = open(newFile, "r")
1417            for line in f.readlines():
1418                newdiff += "+" + line
1419            f.close()
1420
1421        # change description file: submitTemplate, separatorLine, diff, newdiff
1422        (handle, fileName) = tempfile.mkstemp()
1423        tmpFile = os.fdopen(handle, "w+")
1424        if self.isWindows:
1425            submitTemplate = submitTemplate.replace("\n", "\r\n")
1426            separatorLine = separatorLine.replace("\n", "\r\n")
1427            newdiff = newdiff.replace("\n", "\r\n")
1428        tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1429        tmpFile.close()
1430
1431        if self.prepare_p4_only:
1432            #
1433            # Leave the p4 tree prepared, and the submit template around
1434            # and let the user decide what to do next
1435            #
1436            print
1437            print "P4 workspace prepared for submission."
1438            print "To submit or revert, go to client workspace"
1439            print "  " + self.clientPath
1440            print
1441            print "To submit, use \"p4 submit\" to write a new description,"
1442            print "or \"p4 submit -i %s\" to use the one prepared by" \
1443                  " \"git p4\"." % fileName
1444            print "You can delete the file \"%s\" when finished." % fileName
1445
1446            if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1447                print "To preserve change ownership by user %s, you must\n" \
1448                      "do \"p4 change -f <change>\" after submitting and\n" \
1449                      "edit the User field."
1450            if pureRenameCopy:
1451                print "After submitting, renamed files must be re-synced."
1452                print "Invoke \"p4 sync -f\" on each of these files:"
1453                for f in pureRenameCopy:
1454                    print "  " + f
1455
1456            print
1457            print "To revert the changes, use \"p4 revert ...\", and delete"
1458            print "the submit template file \"%s\"" % fileName
1459            if filesToAdd:
1460                print "Since the commit adds new files, they must be deleted:"
1461                for f in filesToAdd:
1462                    print "  " + f
1463            print
1464            return True
1465
1466        #
1467        # Let the user edit the change description, then submit it.
1468        #
1469        if self.edit_template(fileName):
1470            # read the edited message and submit
1471            ret = True
1472            tmpFile = open(fileName, "rb")
1473            message = tmpFile.read()
1474            tmpFile.close()
1475            submitTemplate = message[:message.index(separatorLine)]
1476            if self.isWindows:
1477                submitTemplate = submitTemplate.replace("\r\n", "\n")
1478            p4_write_pipe(['submit', '-i'], submitTemplate)
1479
1480            if self.preserveUser:
1481                if p4User:
1482                    # Get last changelist number. Cannot easily get it from
1483                    # the submit command output as the output is
1484                    # unmarshalled.
1485                    changelist = self.lastP4Changelist()
1486                    self.modifyChangelistUser(changelist, p4User)
1487
1488            # The rename/copy happened by applying a patch that created a
1489            # new file.  This leaves it writable, which confuses p4.
1490            for f in pureRenameCopy:
1491                p4_sync(f, "-f")
1492
1493        else:
1494            # skip this patch
1495            ret = False
1496            print "Submission cancelled, undoing p4 changes."
1497            for f in editedFiles:
1498                p4_revert(f)
1499            for f in filesToAdd:
1500                p4_revert(f)
1501                os.remove(f)
1502            for f in filesToDelete:
1503                p4_revert(f)
1504
1505        os.remove(fileName)
1506        return ret
1507
1508    # Export git tags as p4 labels. Create a p4 label and then tag
1509    # with that.
1510    def exportGitTags(self, gitTags):
1511        validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1512        if len(validLabelRegexp) == 0:
1513            validLabelRegexp = defaultLabelRegexp
1514        m = re.compile(validLabelRegexp)
1515
1516        for name in gitTags:
1517
1518            if not m.match(name):
1519                if verbose:
1520                    print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1521                continue
1522
1523            # Get the p4 commit this corresponds to
1524            logMessage = extractLogMessageFromGitCommit(name)
1525            values = extractSettingsGitLog(logMessage)
1526
1527            if not values.has_key('change'):
1528                # a tag pointing to something not sent to p4; ignore
1529                if verbose:
1530                    print "git tag %s does not give a p4 commit" % name
1531                continue
1532            else:
1533                changelist = values['change']
1534
1535            # Get the tag details.
1536            inHeader = True
1537            isAnnotated = False
1538            body = []
1539            for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1540                l = l.strip()
1541                if inHeader:
1542                    if re.match(r'tag\s+', l):
1543                        isAnnotated = True
1544                    elif re.match(r'\s*$', l):
1545                        inHeader = False
1546                        continue
1547                else:
1548                    body.append(l)
1549
1550            if not isAnnotated:
1551                body = ["lightweight tag imported by git p4\n"]
1552
1553            # Create the label - use the same view as the client spec we are using
1554            clientSpec = getClientSpec()
1555
1556            labelTemplate  = "Label: %s\n" % name
1557            labelTemplate += "Description:\n"
1558            for b in body:
1559                labelTemplate += "\t" + b + "\n"
1560            labelTemplate += "View:\n"
1561            for depot_side in clientSpec.mappings:
1562                labelTemplate += "\t%s\n" % depot_side
1563
1564            if self.dry_run:
1565                print "Would create p4 label %s for tag" % name
1566            elif self.prepare_p4_only:
1567                print "Not creating p4 label %s for tag due to option" \
1568                      " --prepare-p4-only" % name
1569            else:
1570                p4_write_pipe(["label", "-i"], labelTemplate)
1571
1572                # Use the label
1573                p4_system(["tag", "-l", name] +
1574                          ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1575
1576                if verbose:
1577                    print "created p4 label for tag %s" % name
1578
1579    def run(self, args):
1580        if len(args) == 0:
1581            self.master = currentGitBranch()
1582            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1583                die("Detecting current git branch failed!")
1584        elif len(args) == 1:
1585            self.master = args[0]
1586            if not branchExists(self.master):
1587                die("Branch %s does not exist" % self.master)
1588        else:
1589            return False
1590
1591        allowSubmit = gitConfig("git-p4.allowSubmit")
1592        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1593            die("%s is not in git-p4.allowSubmit" % self.master)
1594
1595        [upstream, settings] = findUpstreamBranchPoint()
1596        self.depotPath = settings['depot-paths'][0]
1597        if len(self.origin) == 0:
1598            self.origin = upstream
1599
1600        if self.preserveUser:
1601            if not self.canChangeChangelists():
1602                die("Cannot preserve user names without p4 super-user or admin permissions")
1603
1604        # if not set from the command line, try the config file
1605        if self.conflict_behavior is None:
1606            val = gitConfig("git-p4.conflict")
1607            if val:
1608                if val not in self.conflict_behavior_choices:
1609                    die("Invalid value '%s' for config git-p4.conflict" % val)
1610            else:
1611                val = "ask"
1612            self.conflict_behavior = val
1613
1614        if self.verbose:
1615            print "Origin branch is " + self.origin
1616
1617        if len(self.depotPath) == 0:
1618            print "Internal error: cannot locate perforce depot path from existing branches"
1619            sys.exit(128)
1620
1621        self.useClientSpec = False
1622        if gitConfigBool("git-p4.useclientspec"):
1623            self.useClientSpec = True
1624        if self.useClientSpec:
1625            self.clientSpecDirs = getClientSpec()
1626
1627        if self.useClientSpec:
1628            # all files are relative to the client spec
1629            self.clientPath = getClientRoot()
1630        else:
1631            self.clientPath = p4Where(self.depotPath)
1632
1633        if self.clientPath == "":
1634            die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1635
1636        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1637        self.oldWorkingDirectory = os.getcwd()
1638
1639        # ensure the clientPath exists
1640        new_client_dir = False
1641        if not os.path.exists(self.clientPath):
1642            new_client_dir = True
1643            os.makedirs(self.clientPath)
1644
1645        chdir(self.clientPath, is_client_path=True)
1646        if self.dry_run:
1647            print "Would synchronize p4 checkout in %s" % self.clientPath
1648        else:
1649            print "Synchronizing p4 checkout..."
1650            if new_client_dir:
1651                # old one was destroyed, and maybe nobody told p4
1652                p4_sync("...", "-f")
1653            else:
1654                p4_sync("...")
1655        self.check()
1656
1657        commits = []
1658        for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1659            commits.append(line.strip())
1660        commits.reverse()
1661
1662        if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1663            self.checkAuthorship = False
1664        else:
1665            self.checkAuthorship = True
1666
1667        if self.preserveUser:
1668            self.checkValidP4Users(commits)
1669
1670        #
1671        # Build up a set of options to be passed to diff when
1672        # submitting each commit to p4.
1673        #
1674        if self.detectRenames:
1675            # command-line -M arg
1676            self.diffOpts = "-M"
1677        else:
1678            # If not explicitly set check the config variable
1679            detectRenames = gitConfig("git-p4.detectRenames")
1680
1681            if detectRenames.lower() == "false" or detectRenames == "":
1682                self.diffOpts = ""
1683            elif detectRenames.lower() == "true":
1684                self.diffOpts = "-M"
1685            else:
1686                self.diffOpts = "-M%s" % detectRenames
1687
1688        # no command-line arg for -C or --find-copies-harder, just
1689        # config variables
1690        detectCopies = gitConfig("git-p4.detectCopies")
1691        if detectCopies.lower() == "false" or detectCopies == "":
1692            pass
1693        elif detectCopies.lower() == "true":
1694            self.diffOpts += " -C"
1695        else:
1696            self.diffOpts += " -C%s" % detectCopies
1697
1698        if gitConfigBool("git-p4.detectCopiesHarder"):
1699            self.diffOpts += " --find-copies-harder"
1700
1701        #
1702        # Apply the commits, one at a time.  On failure, ask if should
1703        # continue to try the rest of the patches, or quit.
1704        #
1705        if self.dry_run:
1706            print "Would apply"
1707        applied = []
1708        last = len(commits) - 1
1709        for i, commit in enumerate(commits):
1710            if self.dry_run:
1711                print " ", read_pipe(["git", "show", "-s",
1712                                      "--format=format:%h %s", commit])
1713                ok = True
1714            else:
1715                ok = self.applyCommit(commit)
1716            if ok:
1717                applied.append(commit)
1718            else:
1719                if self.prepare_p4_only and i < last:
1720                    print "Processing only the first commit due to option" \
1721                          " --prepare-p4-only"
1722                    break
1723                if i < last:
1724                    quit = False
1725                    while True:
1726                        # prompt for what to do, or use the option/variable
1727                        if self.conflict_behavior == "ask":
1728                            print "What do you want to do?"
1729                            response = raw_input("[s]kip this commit but apply"
1730                                                 " the rest, or [q]uit? ")
1731                            if not response:
1732                                continue
1733                        elif self.conflict_behavior == "skip":
1734                            response = "s"
1735                        elif self.conflict_behavior == "quit":
1736                            response = "q"
1737                        else:
1738                            die("Unknown conflict_behavior '%s'" %
1739                                self.conflict_behavior)
1740
1741                        if response[0] == "s":
1742                            print "Skipping this commit, but applying the rest"
1743                            break
1744                        if response[0] == "q":
1745                            print "Quitting"
1746                            quit = True
1747                            break
1748                    if quit:
1749                        break
1750
1751        chdir(self.oldWorkingDirectory)
1752
1753        if self.dry_run:
1754            pass
1755        elif self.prepare_p4_only:
1756            pass
1757        elif len(commits) == len(applied):
1758            print "All commits applied!"
1759
1760            sync = P4Sync()
1761            if self.branch:
1762                sync.branch = self.branch
1763            sync.run([])
1764
1765            rebase = P4Rebase()
1766            rebase.rebase()
1767
1768        else:
1769            if len(applied) == 0:
1770                print "No commits applied."
1771            else:
1772                print "Applied only the commits marked with '*':"
1773                for c in commits:
1774                    if c in applied:
1775                        star = "*"
1776                    else:
1777                        star = " "
1778                    print star, read_pipe(["git", "show", "-s",
1779                                           "--format=format:%h %s",  c])
1780                print "You will have to do 'git p4 sync' and rebase."
1781
1782        if gitConfigBool("git-p4.exportLabels"):
1783            self.exportLabels = True
1784
1785        if self.exportLabels:
1786            p4Labels = getP4Labels(self.depotPath)
1787            gitTags = getGitTags()
1788
1789            missingGitTags = gitTags - p4Labels
1790            self.exportGitTags(missingGitTags)
1791
1792        # exit with error unless everything applied perfectly
1793        if len(commits) != len(applied):
1794                sys.exit(1)
1795
1796        return True
1797
1798class View(object):
1799    """Represent a p4 view ("p4 help views"), and map files in a
1800       repo according to the view."""
1801
1802    def __init__(self, client_name):
1803        self.mappings = []
1804        self.client_prefix = "//%s/" % client_name
1805        # cache results of "p4 where" to lookup client file locations
1806        self.client_spec_path_cache = {}
1807
1808    def append(self, view_line):
1809        """Parse a view line, splitting it into depot and client
1810           sides.  Append to self.mappings, preserving order.  This
1811           is only needed for tag creation."""
1812
1813        # Split the view line into exactly two words.  P4 enforces
1814        # structure on these lines that simplifies this quite a bit.
1815        #
1816        # Either or both words may be double-quoted.
1817        # Single quotes do not matter.
1818        # Double-quote marks cannot occur inside the words.
1819        # A + or - prefix is also inside the quotes.
1820        # There are no quotes unless they contain a space.
1821        # The line is already white-space stripped.
1822        # The two words are separated by a single space.
1823        #
1824        if view_line[0] == '"':
1825            # First word is double quoted.  Find its end.
1826            close_quote_index = view_line.find('"', 1)
1827            if close_quote_index <= 0:
1828                die("No first-word closing quote found: %s" % view_line)
1829            depot_side = view_line[1:close_quote_index]
1830            # skip closing quote and space
1831            rhs_index = close_quote_index + 1 + 1
1832        else:
1833            space_index = view_line.find(" ")
1834            if space_index <= 0:
1835                die("No word-splitting space found: %s" % view_line)
1836            depot_side = view_line[0:space_index]
1837            rhs_index = space_index + 1
1838
1839        # prefix + means overlay on previous mapping
1840        if depot_side.startswith("+"):
1841            depot_side = depot_side[1:]
1842
1843        # prefix - means exclude this path, leave out of mappings
1844        exclude = False
1845        if depot_side.startswith("-"):
1846            exclude = True
1847            depot_side = depot_side[1:]
1848
1849        if not exclude:
1850            self.mappings.append(depot_side)
1851
1852    def convert_client_path(self, clientFile):
1853        # chop off //client/ part to make it relative
1854        if not clientFile.startswith(self.client_prefix):
1855            die("No prefix '%s' on clientFile '%s'" %
1856                (self.client_prefix, clientFile))
1857        return clientFile[len(self.client_prefix):]
1858
1859    def update_client_spec_path_cache(self, files):
1860        """ Caching file paths by "p4 where" batch query """
1861
1862        # List depot file paths exclude that already cached
1863        fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1864
1865        if len(fileArgs) == 0:
1866            return  # All files in cache
1867
1868        where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1869        for res in where_result:
1870            if "code" in res and res["code"] == "error":
1871                # assume error is "... file(s) not in client view"
1872                continue
1873            if "clientFile" not in res:
1874                die("No clientFile from 'p4 where %s'" % depot_path)
1875            if "unmap" in res:
1876                # it will list all of them, but only one not unmap-ped
1877                continue
1878            self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1879
1880        # not found files or unmap files set to ""
1881        for depotFile in fileArgs:
1882            if depotFile not in self.client_spec_path_cache:
1883                self.client_spec_path_cache[depotFile] = ""
1884
1885    def map_in_client(self, depot_path):
1886        """Return the relative location in the client where this
1887           depot file should live.  Returns "" if the file should
1888           not be mapped in the client."""
1889
1890        if depot_path in self.client_spec_path_cache:
1891            return self.client_spec_path_cache[depot_path]
1892
1893        die( "Error: %s is not found in client spec path" % depot_path )
1894        return ""
1895
1896class P4Sync(Command, P4UserMap):
1897    delete_actions = ( "delete", "move/delete", "purge" )
1898
1899    def __init__(self):
1900        Command.__init__(self)
1901        P4UserMap.__init__(self)
1902        self.options = [
1903                optparse.make_option("--branch", dest="branch"),
1904                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1905                optparse.make_option("--changesfile", dest="changesFile"),
1906                optparse.make_option("--silent", dest="silent", action="store_true"),
1907                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1908                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1909                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1910                                     help="Import into refs/heads/ , not refs/remotes"),
1911                optparse.make_option("--max-changes", dest="maxChanges"),
1912                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1913                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1914                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1915                                     help="Only sync files that are included in the Perforce Client Spec")
1916        ]
1917        self.description = """Imports from Perforce into a git repository.\n
1918    example:
1919    //depot/my/project/ -- to import the current head
1920    //depot/my/project/@all -- to import everything
1921    //depot/my/project/@1,6 -- to import only from revision 1 to 6
1922
1923    (a ... is not needed in the path p4 specification, it's added implicitly)"""
1924
1925        self.usage += " //depot/path[@revRange]"
1926        self.silent = False
1927        self.createdBranches = set()
1928        self.committedChanges = set()
1929        self.branch = ""
1930        self.detectBranches = False
1931        self.detectLabels = False
1932        self.importLabels = False
1933        self.changesFile = ""
1934        self.syncWithOrigin = True
1935        self.importIntoRemotes = True
1936        self.maxChanges = ""
1937        self.keepRepoPath = False
1938        self.depotPaths = None
1939        self.p4BranchesInGit = []
1940        self.cloneExclude = []
1941        self.useClientSpec = False
1942        self.useClientSpec_from_options = False
1943        self.clientSpecDirs = None
1944        self.tempBranches = []
1945        self.tempBranchLocation = "git-p4-tmp"
1946
1947        if gitConfig("git-p4.syncFromOrigin") == "false":
1948            self.syncWithOrigin = False
1949
1950    # Force a checkpoint in fast-import and wait for it to finish
1951    def checkpoint(self):
1952        self.gitStream.write("checkpoint\n\n")
1953        self.gitStream.write("progress checkpoint\n\n")
1954        out = self.gitOutput.readline()
1955        if self.verbose:
1956            print "checkpoint finished: " + out
1957
1958    def extractFilesFromCommit(self, commit):
1959        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1960                             for path in self.cloneExclude]
1961        files = []
1962        fnum = 0
1963        while commit.has_key("depotFile%s" % fnum):
1964            path =  commit["depotFile%s" % fnum]
1965
1966            if [p for p in self.cloneExclude
1967                if p4PathStartsWith(path, p)]:
1968                found = False
1969            else:
1970                found = [p for p in self.depotPaths
1971                         if p4PathStartsWith(path, p)]
1972            if not found:
1973                fnum = fnum + 1
1974                continue
1975
1976            file = {}
1977            file["path"] = path
1978            file["rev"] = commit["rev%s" % fnum]
1979            file["action"] = commit["action%s" % fnum]
1980            file["type"] = commit["type%s" % fnum]
1981            files.append(file)
1982            fnum = fnum + 1
1983        return files
1984
1985    def stripRepoPath(self, path, prefixes):
1986        """When streaming files, this is called to map a p4 depot path
1987           to where it should go in git.  The prefixes are either
1988           self.depotPaths, or self.branchPrefixes in the case of
1989           branch detection."""
1990
1991        if self.useClientSpec:
1992            # branch detection moves files up a level (the branch name)
1993            # from what client spec interpretation gives
1994            path = self.clientSpecDirs.map_in_client(path)
1995            if self.detectBranches:
1996                for b in self.knownBranches:
1997                    if path.startswith(b + "/"):
1998                        path = path[len(b)+1:]
1999
2000        elif self.keepRepoPath:
2001            # Preserve everything in relative path name except leading
2002            # //depot/; just look at first prefix as they all should
2003            # be in the same depot.
2004            depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2005            if p4PathStartsWith(path, depot):
2006                path = path[len(depot):]
2007
2008        else:
2009            for p in prefixes:
2010                if p4PathStartsWith(path, p):
2011                    path = path[len(p):]
2012                    break
2013
2014        path = wildcard_decode(path)
2015        return path
2016
2017    def splitFilesIntoBranches(self, commit):
2018        """Look at each depotFile in the commit to figure out to what
2019           branch it belongs."""
2020
2021        if self.clientSpecDirs:
2022            files = self.extractFilesFromCommit(commit)
2023            self.clientSpecDirs.update_client_spec_path_cache(files)
2024
2025        branches = {}
2026        fnum = 0
2027        while commit.has_key("depotFile%s" % fnum):
2028            path =  commit["depotFile%s" % fnum]
2029            found = [p for p in self.depotPaths
2030                     if p4PathStartsWith(path, p)]
2031            if not found:
2032                fnum = fnum + 1
2033                continue
2034
2035            file = {}
2036            file["path"] = path
2037            file["rev"] = commit["rev%s" % fnum]
2038            file["action"] = commit["action%s" % fnum]
2039            file["type"] = commit["type%s" % fnum]
2040            fnum = fnum + 1
2041
2042            # start with the full relative path where this file would
2043            # go in a p4 client
2044            if self.useClientSpec:
2045                relPath = self.clientSpecDirs.map_in_client(path)
2046            else:
2047                relPath = self.stripRepoPath(path, self.depotPaths)
2048
2049            for branch in self.knownBranches.keys():
2050                # add a trailing slash so that a commit into qt/4.2foo
2051                # doesn't end up in qt/4.2, e.g.
2052                if relPath.startswith(branch + "/"):
2053                    if branch not in branches:
2054                        branches[branch] = []
2055                    branches[branch].append(file)
2056                    break
2057
2058        return branches
2059
2060    # output one file from the P4 stream
2061    # - helper for streamP4Files
2062
2063    def streamOneP4File(self, file, contents):
2064        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2065        if verbose:
2066            sys.stderr.write("%s\n" % relPath)
2067
2068        (type_base, type_mods) = split_p4_type(file["type"])
2069
2070        git_mode = "100644"
2071        if "x" in type_mods:
2072            git_mode = "100755"
2073        if type_base == "symlink":
2074            git_mode = "120000"
2075            # p4 print on a symlink sometimes contains "target\n";
2076            # if it does, remove the newline
2077            data = ''.join(contents)
2078            if data[-1] == '\n':
2079                contents = [data[:-1]]
2080            else:
2081                contents = [data]
2082
2083        if type_base == "utf16":
2084            # p4 delivers different text in the python output to -G
2085            # than it does when using "print -o", or normal p4 client
2086            # operations.  utf16 is converted to ascii or utf8, perhaps.
2087            # But ascii text saved as -t utf16 is completely mangled.
2088            # Invoke print -o to get the real contents.
2089            #
2090            # On windows, the newlines will always be mangled by print, so put
2091            # them back too.  This is not needed to the cygwin windows version,
2092            # just the native "NT" type.
2093            #
2094            text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2095            if p4_version_string().find("/NT") >= 0:
2096                text = text.replace("\r\n", "\n")
2097            contents = [ text ]
2098
2099        if type_base == "apple":
2100            # Apple filetype files will be streamed as a concatenation of
2101            # its appledouble header and the contents.  This is useless
2102            # on both macs and non-macs.  If using "print -q -o xx", it
2103            # will create "xx" with the data, and "%xx" with the header.
2104            # This is also not very useful.
2105            #
2106            # Ideally, someday, this script can learn how to generate
2107            # appledouble files directly and import those to git, but
2108            # non-mac machines can never find a use for apple filetype.
2109            print "\nIgnoring apple filetype file %s" % file['depotFile']
2110            return
2111
2112        # Note that we do not try to de-mangle keywords on utf16 files,
2113        # even though in theory somebody may want that.
2114        pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2115        if pattern:
2116            regexp = re.compile(pattern, re.VERBOSE)
2117            text = ''.join(contents)
2118            text = regexp.sub(r'$\1$', text)
2119            contents = [ text ]
2120
2121        self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2122
2123        # total length...
2124        length = 0
2125        for d in contents:
2126            length = length + len(d)
2127
2128        self.gitStream.write("data %d\n" % length)
2129        for d in contents:
2130            self.gitStream.write(d)
2131        self.gitStream.write("\n")
2132
2133    def streamOneP4Deletion(self, file):
2134        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2135        if verbose:
2136            sys.stderr.write("delete %s\n" % relPath)
2137        self.gitStream.write("D %s\n" % relPath)
2138
2139    # handle another chunk of streaming data
2140    def streamP4FilesCb(self, marshalled):
2141
2142        # catch p4 errors and complain
2143        err = None
2144        if "code" in marshalled:
2145            if marshalled["code"] == "error":
2146                if "data" in marshalled:
2147                    err = marshalled["data"].rstrip()
2148        if err:
2149            f = None
2150            if self.stream_have_file_info:
2151                if "depotFile" in self.stream_file:
2152                    f = self.stream_file["depotFile"]
2153            # force a failure in fast-import, else an empty
2154            # commit will be made
2155            self.gitStream.write("\n")
2156            self.gitStream.write("die-now\n")
2157            self.gitStream.close()
2158            # ignore errors, but make sure it exits first
2159            self.importProcess.wait()
2160            if f:
2161                die("Error from p4 print for %s: %s" % (f, err))
2162            else:
2163                die("Error from p4 print: %s" % err)
2164
2165        if marshalled.has_key('depotFile') and self.stream_have_file_info:
2166            # start of a new file - output the old one first
2167            self.streamOneP4File(self.stream_file, self.stream_contents)
2168            self.stream_file = {}
2169            self.stream_contents = []
2170            self.stream_have_file_info = False
2171
2172        # pick up the new file information... for the
2173        # 'data' field we need to append to our array
2174        for k in marshalled.keys():
2175            if k == 'data':
2176                self.stream_contents.append(marshalled['data'])
2177            else:
2178                self.stream_file[k] = marshalled[k]
2179
2180        self.stream_have_file_info = True
2181
2182    # Stream directly from "p4 files" into "git fast-import"
2183    def streamP4Files(self, files):
2184        filesForCommit = []
2185        filesToRead = []
2186        filesToDelete = []
2187
2188        for f in files:
2189            # if using a client spec, only add the files that have
2190            # a path in the client
2191            if self.clientSpecDirs:
2192                if self.clientSpecDirs.map_in_client(f['path']) == "":
2193                    continue
2194
2195            filesForCommit.append(f)
2196            if f['action'] in self.delete_actions:
2197                filesToDelete.append(f)
2198            else:
2199                filesToRead.append(f)
2200
2201        # deleted files...
2202        for f in filesToDelete:
2203            self.streamOneP4Deletion(f)
2204
2205        if len(filesToRead) > 0:
2206            self.stream_file = {}
2207            self.stream_contents = []
2208            self.stream_have_file_info = False
2209
2210            # curry self argument
2211            def streamP4FilesCbSelf(entry):
2212                self.streamP4FilesCb(entry)
2213
2214            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2215
2216            p4CmdList(["-x", "-", "print"],
2217                      stdin=fileArgs,
2218                      cb=streamP4FilesCbSelf)
2219
2220            # do the last chunk
2221            if self.stream_file.has_key('depotFile'):
2222                self.streamOneP4File(self.stream_file, self.stream_contents)
2223
2224    def make_email(self, userid):
2225        if userid in self.users:
2226            return self.users[userid]
2227        else:
2228            return "%s <a@b>" % userid
2229
2230    # Stream a p4 tag
2231    def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2232        if verbose:
2233            print "writing tag %s for commit %s" % (labelName, commit)
2234        gitStream.write("tag %s\n" % labelName)
2235        gitStream.write("from %s\n" % commit)
2236
2237        if labelDetails.has_key('Owner'):
2238            owner = labelDetails["Owner"]
2239        else:
2240            owner = None
2241
2242        # Try to use the owner of the p4 label, or failing that,
2243        # the current p4 user id.
2244        if owner:
2245            email = self.make_email(owner)
2246        else:
2247            email = self.make_email(self.p4UserId())
2248        tagger = "%s %s %s" % (email, epoch, self.tz)
2249
2250        gitStream.write("tagger %s\n" % tagger)
2251
2252        print "labelDetails=",labelDetails
2253        if labelDetails.has_key('Description'):
2254            description = labelDetails['Description']
2255        else:
2256            description = 'Label from git p4'
2257
2258        gitStream.write("data %d\n" % len(description))
2259        gitStream.write(description)
2260        gitStream.write("\n")
2261
2262    def commit(self, details, files, branch, parent = ""):
2263        epoch = details["time"]
2264        author = details["user"]
2265
2266        if self.verbose:
2267            print "commit into %s" % branch
2268
2269        # start with reading files; if that fails, we should not
2270        # create a commit.
2271        new_files = []
2272        for f in files:
2273            if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2274                new_files.append (f)
2275            else:
2276                sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2277
2278        if self.clientSpecDirs:
2279            self.clientSpecDirs.update_client_spec_path_cache(files)
2280
2281        self.gitStream.write("commit %s\n" % branch)
2282#        gitStream.write("mark :%s\n" % details["change"])
2283        self.committedChanges.add(int(details["change"]))
2284        committer = ""
2285        if author not in self.users:
2286            self.getUserMapFromPerforceServer()
2287        committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2288
2289        self.gitStream.write("committer %s\n" % committer)
2290
2291        self.gitStream.write("data <<EOT\n")
2292        self.gitStream.write(details["desc"])
2293        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2294                             (','.join(self.branchPrefixes), details["change"]))
2295        if len(details['options']) > 0:
2296            self.gitStream.write(": options = %s" % details['options'])
2297        self.gitStream.write("]\nEOT\n\n")
2298
2299        if len(parent) > 0:
2300            if self.verbose:
2301                print "parent %s" % parent
2302            self.gitStream.write("from %s\n" % parent)
2303
2304        self.streamP4Files(new_files)
2305        self.gitStream.write("\n")
2306
2307        change = int(details["change"])
2308
2309        if self.labels.has_key(change):
2310            label = self.labels[change]
2311            labelDetails = label[0]
2312            labelRevisions = label[1]
2313            if self.verbose:
2314                print "Change %s is labelled %s" % (change, labelDetails)
2315
2316            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2317                                                for p in self.branchPrefixes])
2318
2319            if len(files) == len(labelRevisions):
2320
2321                cleanedFiles = {}
2322                for info in files:
2323                    if info["action"] in self.delete_actions:
2324                        continue
2325                    cleanedFiles[info["depotFile"]] = info["rev"]
2326
2327                if cleanedFiles == labelRevisions:
2328                    self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2329
2330                else:
2331                    if not self.silent:
2332                        print ("Tag %s does not match with change %s: files do not match."
2333                               % (labelDetails["label"], change))
2334
2335            else:
2336                if not self.silent:
2337                    print ("Tag %s does not match with change %s: file count is different."
2338                           % (labelDetails["label"], change))
2339
2340    # Build a dictionary of changelists and labels, for "detect-labels" option.
2341    def getLabels(self):
2342        self.labels = {}
2343
2344        l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2345        if len(l) > 0 and not self.silent:
2346            print "Finding files belonging to labels in %s" % `self.depotPaths`
2347
2348        for output in l:
2349            label = output["label"]
2350            revisions = {}
2351            newestChange = 0
2352            if self.verbose:
2353                print "Querying files for label %s" % label
2354            for file in p4CmdList(["files"] +
2355                                      ["%s...@%s" % (p, label)
2356                                          for p in self.depotPaths]):
2357                revisions[file["depotFile"]] = file["rev"]
2358                change = int(file["change"])
2359                if change > newestChange:
2360                    newestChange = change
2361
2362            self.labels[newestChange] = [output, revisions]
2363
2364        if self.verbose:
2365            print "Label changes: %s" % self.labels.keys()
2366
2367    # Import p4 labels as git tags. A direct mapping does not
2368    # exist, so assume that if all the files are at the same revision
2369    # then we can use that, or it's something more complicated we should
2370    # just ignore.
2371    def importP4Labels(self, stream, p4Labels):
2372        if verbose:
2373            print "import p4 labels: " + ' '.join(p4Labels)
2374
2375        ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2376        validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2377        if len(validLabelRegexp) == 0:
2378            validLabelRegexp = defaultLabelRegexp
2379        m = re.compile(validLabelRegexp)
2380
2381        for name in p4Labels:
2382            commitFound = False
2383
2384            if not m.match(name):
2385                if verbose:
2386                    print "label %s does not match regexp %s" % (name,validLabelRegexp)
2387                continue
2388
2389            if name in ignoredP4Labels:
2390                continue
2391
2392            labelDetails = p4CmdList(['label', "-o", name])[0]
2393
2394            # get the most recent changelist for each file in this label
2395            change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2396                                for p in self.depotPaths])
2397
2398            if change.has_key('change'):
2399                # find the corresponding git commit; take the oldest commit
2400                changelist = int(change['change'])
2401                gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2402                     "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2403                if len(gitCommit) == 0:
2404                    print "could not find git commit for changelist %d" % changelist
2405                else:
2406                    gitCommit = gitCommit.strip()
2407                    commitFound = True
2408                    # Convert from p4 time format
2409                    try:
2410                        tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2411                    except ValueError:
2412                        print "Could not convert label time %s" % labelDetails['Update']
2413                        tmwhen = 1
2414
2415                    when = int(time.mktime(tmwhen))
2416                    self.streamTag(stream, name, labelDetails, gitCommit, when)
2417                    if verbose:
2418                        print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2419            else:
2420                if verbose:
2421                    print "Label %s has no changelists - possibly deleted?" % name
2422
2423            if not commitFound:
2424                # We can't import this label; don't try again as it will get very
2425                # expensive repeatedly fetching all the files for labels that will
2426                # never be imported. If the label is moved in the future, the
2427                # ignore will need to be removed manually.
2428                system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2429
2430    def guessProjectName(self):
2431        for p in self.depotPaths:
2432            if p.endswith("/"):
2433                p = p[:-1]
2434            p = p[p.strip().rfind("/") + 1:]
2435            if not p.endswith("/"):
2436               p += "/"
2437            return p
2438
2439    def getBranchMapping(self):
2440        lostAndFoundBranches = set()
2441
2442        user = gitConfig("git-p4.branchUser")
2443        if len(user) > 0:
2444            command = "branches -u %s" % user
2445        else:
2446            command = "branches"
2447
2448        for info in p4CmdList(command):
2449            details = p4Cmd(["branch", "-o", info["branch"]])
2450            viewIdx = 0
2451            while details.has_key("View%s" % viewIdx):
2452                paths = details["View%s" % viewIdx].split(" ")
2453                viewIdx = viewIdx + 1
2454                # require standard //depot/foo/... //depot/bar/... mapping
2455                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2456                    continue
2457                source = paths[0]
2458                destination = paths[1]
2459                ## HACK
2460                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2461                    source = source[len(self.depotPaths[0]):-4]
2462                    destination = destination[len(self.depotPaths[0]):-4]
2463
2464                    if destination in self.knownBranches:
2465                        if not self.silent:
2466                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2467                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2468                        continue
2469
2470                    self.knownBranches[destination] = source
2471
2472                    lostAndFoundBranches.discard(destination)
2473
2474                    if source not in self.knownBranches:
2475                        lostAndFoundBranches.add(source)
2476
2477        # Perforce does not strictly require branches to be defined, so we also
2478        # check git config for a branch list.
2479        #
2480        # Example of branch definition in git config file:
2481        # [git-p4]
2482        #   branchList=main:branchA
2483        #   branchList=main:branchB
2484        #   branchList=branchA:branchC
2485        configBranches = gitConfigList("git-p4.branchList")
2486        for branch in configBranches:
2487            if branch:
2488                (source, destination) = branch.split(":")
2489                self.knownBranches[destination] = source
2490
2491                lostAndFoundBranches.discard(destination)
2492
2493                if source not in self.knownBranches:
2494                    lostAndFoundBranches.add(source)
2495
2496
2497        for branch in lostAndFoundBranches:
2498            self.knownBranches[branch] = branch
2499
2500    def getBranchMappingFromGitBranches(self):
2501        branches = p4BranchesInGit(self.importIntoRemotes)
2502        for branch in branches.keys():
2503            if branch == "master":
2504                branch = "main"
2505            else:
2506                branch = branch[len(self.projectName):]
2507            self.knownBranches[branch] = branch
2508
2509    def updateOptionDict(self, d):
2510        option_keys = {}
2511        if self.keepRepoPath:
2512            option_keys['keepRepoPath'] = 1
2513
2514        d["options"] = ' '.join(sorted(option_keys.keys()))
2515
2516    def readOptions(self, d):
2517        self.keepRepoPath = (d.has_key('options')
2518                             and ('keepRepoPath' in d['options']))
2519
2520    def gitRefForBranch(self, branch):
2521        if branch == "main":
2522            return self.refPrefix + "master"
2523
2524        if len(branch) <= 0:
2525            return branch
2526
2527        return self.refPrefix + self.projectName + branch
2528
2529    def gitCommitByP4Change(self, ref, change):
2530        if self.verbose:
2531            print "looking in ref " + ref + " for change %s using bisect..." % change
2532
2533        earliestCommit = ""
2534        latestCommit = parseRevision(ref)
2535
2536        while True:
2537            if self.verbose:
2538                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2539            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2540            if len(next) == 0:
2541                if self.verbose:
2542                    print "argh"
2543                return ""
2544            log = extractLogMessageFromGitCommit(next)
2545            settings = extractSettingsGitLog(log)
2546            currentChange = int(settings['change'])
2547            if self.verbose:
2548                print "current change %s" % currentChange
2549
2550            if currentChange == change:
2551                if self.verbose:
2552                    print "found %s" % next
2553                return next
2554
2555            if currentChange < change:
2556                earliestCommit = "^%s" % next
2557            else:
2558                latestCommit = "%s" % next
2559
2560        return ""
2561
2562    def importNewBranch(self, branch, maxChange):
2563        # make fast-import flush all changes to disk and update the refs using the checkpoint
2564        # command so that we can try to find the branch parent in the git history
2565        self.gitStream.write("checkpoint\n\n");
2566        self.gitStream.flush();
2567        branchPrefix = self.depotPaths[0] + branch + "/"
2568        range = "@1,%s" % maxChange
2569        #print "prefix" + branchPrefix
2570        changes = p4ChangesForPaths([branchPrefix], range)
2571        if len(changes) <= 0:
2572            return False
2573        firstChange = changes[0]
2574        #print "first change in branch: %s" % firstChange
2575        sourceBranch = self.knownBranches[branch]
2576        sourceDepotPath = self.depotPaths[0] + sourceBranch
2577        sourceRef = self.gitRefForBranch(sourceBranch)
2578        #print "source " + sourceBranch
2579
2580        branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2581        #print "branch parent: %s" % branchParentChange
2582        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2583        if len(gitParent) > 0:
2584            self.initialParents[self.gitRefForBranch(branch)] = gitParent
2585            #print "parent git commit: %s" % gitParent
2586
2587        self.importChanges(changes)
2588        return True
2589
2590    def searchParent(self, parent, branch, target):
2591        parentFound = False
2592        for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2593                                     "--no-merges", parent]):
2594            blob = blob.strip()
2595            if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2596                parentFound = True
2597                if self.verbose:
2598                    print "Found parent of %s in commit %s" % (branch, blob)
2599                break
2600        if parentFound:
2601            return blob
2602        else:
2603            return None
2604
2605    def importChanges(self, changes):
2606        cnt = 1
2607        for change in changes:
2608            description = p4_describe(change)
2609            self.updateOptionDict(description)
2610
2611            if not self.silent:
2612                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2613                sys.stdout.flush()
2614            cnt = cnt + 1
2615
2616            try:
2617                if self.detectBranches:
2618                    branches = self.splitFilesIntoBranches(description)
2619                    for branch in branches.keys():
2620                        ## HACK  --hwn
2621                        branchPrefix = self.depotPaths[0] + branch + "/"
2622                        self.branchPrefixes = [ branchPrefix ]
2623
2624                        parent = ""
2625
2626                        filesForCommit = branches[branch]
2627
2628                        if self.verbose:
2629                            print "branch is %s" % branch
2630
2631                        self.updatedBranches.add(branch)
2632
2633                        if branch not in self.createdBranches:
2634                            self.createdBranches.add(branch)
2635                            parent = self.knownBranches[branch]
2636                            if parent == branch:
2637                                parent = ""
2638                            else:
2639                                fullBranch = self.projectName + branch
2640                                if fullBranch not in self.p4BranchesInGit:
2641                                    if not self.silent:
2642                                        print("\n    Importing new branch %s" % fullBranch);
2643                                    if self.importNewBranch(branch, change - 1):
2644                                        parent = ""
2645                                        self.p4BranchesInGit.append(fullBranch)
2646                                    if not self.silent:
2647                                        print("\n    Resuming with change %s" % change);
2648
2649                                if self.verbose:
2650                                    print "parent determined through known branches: %s" % parent
2651
2652                        branch = self.gitRefForBranch(branch)
2653                        parent = self.gitRefForBranch(parent)
2654
2655                        if self.verbose:
2656                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2657
2658                        if len(parent) == 0 and branch in self.initialParents:
2659                            parent = self.initialParents[branch]
2660                            del self.initialParents[branch]
2661
2662                        blob = None
2663                        if len(parent) > 0:
2664                            tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2665                            if self.verbose:
2666                                print "Creating temporary branch: " + tempBranch
2667                            self.commit(description, filesForCommit, tempBranch)
2668                            self.tempBranches.append(tempBranch)
2669                            self.checkpoint()
2670                            blob = self.searchParent(parent, branch, tempBranch)
2671                        if blob:
2672                            self.commit(description, filesForCommit, branch, blob)
2673                        else:
2674                            if self.verbose:
2675                                print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2676                            self.commit(description, filesForCommit, branch, parent)
2677                else:
2678                    files = self.extractFilesFromCommit(description)
2679                    self.commit(description, files, self.branch,
2680                                self.initialParent)
2681                    # only needed once, to connect to the previous commit
2682                    self.initialParent = ""
2683            except IOError:
2684                print self.gitError.read()
2685                sys.exit(1)
2686
2687    def importHeadRevision(self, revision):
2688        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2689
2690        details = {}
2691        details["user"] = "git perforce import user"
2692        details["desc"] = ("Initial import of %s from the state at revision %s\n"
2693                           % (' '.join(self.depotPaths), revision))
2694        details["change"] = revision
2695        newestRevision = 0
2696
2697        fileCnt = 0
2698        fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2699
2700        for info in p4CmdList(["files"] + fileArgs):
2701
2702            if 'code' in info and info['code'] == 'error':
2703                sys.stderr.write("p4 returned an error: %s\n"
2704                                 % info['data'])
2705                if info['data'].find("must refer to client") >= 0:
2706                    sys.stderr.write("This particular p4 error is misleading.\n")
2707                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
2708                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2709                sys.exit(1)
2710            if 'p4ExitCode' in info:
2711                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2712                sys.exit(1)
2713
2714
2715            change = int(info["change"])
2716            if change > newestRevision:
2717                newestRevision = change
2718
2719            if info["action"] in self.delete_actions:
2720                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2721                #fileCnt = fileCnt + 1
2722                continue
2723
2724            for prop in ["depotFile", "rev", "action", "type" ]:
2725                details["%s%s" % (prop, fileCnt)] = info[prop]
2726
2727            fileCnt = fileCnt + 1
2728
2729        details["change"] = newestRevision
2730
2731        # Use time from top-most change so that all git p4 clones of
2732        # the same p4 repo have the same commit SHA1s.
2733        res = p4_describe(newestRevision)
2734        details["time"] = res["time"]
2735
2736        self.updateOptionDict(details)
2737        try:
2738            self.commit(details, self.extractFilesFromCommit(details), self.branch)
2739        except IOError:
2740            print "IO error with git fast-import. Is your git version recent enough?"
2741            print self.gitError.read()
2742
2743
2744    def run(self, args):
2745        self.depotPaths = []
2746        self.changeRange = ""
2747        self.previousDepotPaths = []
2748        self.hasOrigin = False
2749
2750        # map from branch depot path to parent branch
2751        self.knownBranches = {}
2752        self.initialParents = {}
2753
2754        if self.importIntoRemotes:
2755            self.refPrefix = "refs/remotes/p4/"
2756        else:
2757            self.refPrefix = "refs/heads/p4/"
2758
2759        if self.syncWithOrigin:
2760            self.hasOrigin = originP4BranchesExist()
2761            if self.hasOrigin:
2762                if not self.silent:
2763                    print 'Syncing with origin first, using "git fetch origin"'
2764                system("git fetch origin")
2765
2766        branch_arg_given = bool(self.branch)
2767        if len(self.branch) == 0:
2768            self.branch = self.refPrefix + "master"
2769            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2770                system("git update-ref %s refs/heads/p4" % self.branch)
2771                system("git branch -D p4")
2772
2773        # accept either the command-line option, or the configuration variable
2774        if self.useClientSpec:
2775            # will use this after clone to set the variable
2776            self.useClientSpec_from_options = True
2777        else:
2778            if gitConfigBool("git-p4.useclientspec"):
2779                self.useClientSpec = True
2780        if self.useClientSpec:
2781            self.clientSpecDirs = getClientSpec()
2782
2783        # TODO: should always look at previous commits,
2784        # merge with previous imports, if possible.
2785        if args == []:
2786            if self.hasOrigin:
2787                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2788
2789            # branches holds mapping from branch name to sha1
2790            branches = p4BranchesInGit(self.importIntoRemotes)
2791
2792            # restrict to just this one, disabling detect-branches
2793            if branch_arg_given:
2794                short = self.branch.split("/")[-1]
2795                if short in branches:
2796                    self.p4BranchesInGit = [ short ]
2797            else:
2798                self.p4BranchesInGit = branches.keys()
2799
2800            if len(self.p4BranchesInGit) > 1:
2801                if not self.silent:
2802                    print "Importing from/into multiple branches"
2803                self.detectBranches = True
2804                for branch in branches.keys():
2805                    self.initialParents[self.refPrefix + branch] = \
2806                        branches[branch]
2807
2808            if self.verbose:
2809                print "branches: %s" % self.p4BranchesInGit
2810
2811            p4Change = 0
2812            for branch in self.p4BranchesInGit:
2813                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2814
2815                settings = extractSettingsGitLog(logMsg)
2816
2817                self.readOptions(settings)
2818                if (settings.has_key('depot-paths')
2819                    and settings.has_key ('change')):
2820                    change = int(settings['change']) + 1
2821                    p4Change = max(p4Change, change)
2822
2823                    depotPaths = sorted(settings['depot-paths'])
2824                    if self.previousDepotPaths == []:
2825                        self.previousDepotPaths = depotPaths
2826                    else:
2827                        paths = []
2828                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2829                            prev_list = prev.split("/")
2830                            cur_list = cur.split("/")
2831                            for i in range(0, min(len(cur_list), len(prev_list))):
2832                                if cur_list[i] <> prev_list[i]:
2833                                    i = i - 1
2834                                    break
2835
2836                            paths.append ("/".join(cur_list[:i + 1]))
2837
2838                        self.previousDepotPaths = paths
2839
2840            if p4Change > 0:
2841                self.depotPaths = sorted(self.previousDepotPaths)
2842                self.changeRange = "@%s,#head" % p4Change
2843                if not self.silent and not self.detectBranches:
2844                    print "Performing incremental import into %s git branch" % self.branch
2845
2846        # accept multiple ref name abbreviations:
2847        #    refs/foo/bar/branch -> use it exactly
2848        #    p4/branch -> prepend refs/remotes/ or refs/heads/
2849        #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2850        if not self.branch.startswith("refs/"):
2851            if self.importIntoRemotes:
2852                prepend = "refs/remotes/"
2853            else:
2854                prepend = "refs/heads/"
2855            if not self.branch.startswith("p4/"):
2856                prepend += "p4/"
2857            self.branch = prepend + self.branch
2858
2859        if len(args) == 0 and self.depotPaths:
2860            if not self.silent:
2861                print "Depot paths: %s" % ' '.join(self.depotPaths)
2862        else:
2863            if self.depotPaths and self.depotPaths != args:
2864                print ("previous import used depot path %s and now %s was specified. "
2865                       "This doesn't work!" % (' '.join (self.depotPaths),
2866                                               ' '.join (args)))
2867                sys.exit(1)
2868
2869            self.depotPaths = sorted(args)
2870
2871        revision = ""
2872        self.users = {}
2873
2874        # Make sure no revision specifiers are used when --changesfile
2875        # is specified.
2876        bad_changesfile = False
2877        if len(self.changesFile) > 0:
2878            for p in self.depotPaths:
2879                if p.find("@") >= 0 or p.find("#") >= 0:
2880                    bad_changesfile = True
2881                    break
2882        if bad_changesfile:
2883            die("Option --changesfile is incompatible with revision specifiers")
2884
2885        newPaths = []
2886        for p in self.depotPaths:
2887            if p.find("@") != -1:
2888                atIdx = p.index("@")
2889                self.changeRange = p[atIdx:]
2890                if self.changeRange == "@all":
2891                    self.changeRange = ""
2892                elif ',' not in self.changeRange:
2893                    revision = self.changeRange
2894                    self.changeRange = ""
2895                p = p[:atIdx]
2896            elif p.find("#") != -1:
2897                hashIdx = p.index("#")
2898                revision = p[hashIdx:]
2899                p = p[:hashIdx]
2900            elif self.previousDepotPaths == []:
2901                # pay attention to changesfile, if given, else import
2902                # the entire p4 tree at the head revision
2903                if len(self.changesFile) == 0:
2904                    revision = "#head"
2905
2906            p = re.sub ("\.\.\.$", "", p)
2907            if not p.endswith("/"):
2908                p += "/"
2909
2910            newPaths.append(p)
2911
2912        self.depotPaths = newPaths
2913
2914        # --detect-branches may change this for each branch
2915        self.branchPrefixes = self.depotPaths
2916
2917        self.loadUserMapFromCache()
2918        self.labels = {}
2919        if self.detectLabels:
2920            self.getLabels();
2921
2922        if self.detectBranches:
2923            ## FIXME - what's a P4 projectName ?
2924            self.projectName = self.guessProjectName()
2925
2926            if self.hasOrigin:
2927                self.getBranchMappingFromGitBranches()
2928            else:
2929                self.getBranchMapping()
2930            if self.verbose:
2931                print "p4-git branches: %s" % self.p4BranchesInGit
2932                print "initial parents: %s" % self.initialParents
2933            for b in self.p4BranchesInGit:
2934                if b != "master":
2935
2936                    ## FIXME
2937                    b = b[len(self.projectName):]
2938                self.createdBranches.add(b)
2939
2940        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2941
2942        self.importProcess = subprocess.Popen(["git", "fast-import"],
2943                                              stdin=subprocess.PIPE,
2944                                              stdout=subprocess.PIPE,
2945                                              stderr=subprocess.PIPE);
2946        self.gitOutput = self.importProcess.stdout
2947        self.gitStream = self.importProcess.stdin
2948        self.gitError = self.importProcess.stderr
2949
2950        if revision:
2951            self.importHeadRevision(revision)
2952        else:
2953            changes = []
2954
2955            if len(self.changesFile) > 0:
2956                output = open(self.changesFile).readlines()
2957                changeSet = set()
2958                for line in output:
2959                    changeSet.add(int(line))
2960
2961                for change in changeSet:
2962                    changes.append(change)
2963
2964                changes.sort()
2965            else:
2966                # catch "git p4 sync" with no new branches, in a repo that
2967                # does not have any existing p4 branches
2968                if len(args) == 0:
2969                    if not self.p4BranchesInGit:
2970                        die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
2971
2972                    # The default branch is master, unless --branch is used to
2973                    # specify something else.  Make sure it exists, or complain
2974                    # nicely about how to use --branch.
2975                    if not self.detectBranches:
2976                        if not branch_exists(self.branch):
2977                            if branch_arg_given:
2978                                die("Error: branch %s does not exist." % self.branch)
2979                            else:
2980                                die("Error: no branch %s; perhaps specify one with --branch." %
2981                                    self.branch)
2982
2983                if self.verbose:
2984                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2985                                                              self.changeRange)
2986                changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2987
2988                if len(self.maxChanges) > 0:
2989                    changes = changes[:min(int(self.maxChanges), len(changes))]
2990
2991            if len(changes) == 0:
2992                if not self.silent:
2993                    print "No changes to import!"
2994            else:
2995                if not self.silent and not self.detectBranches:
2996                    print "Import destination: %s" % self.branch
2997
2998                self.updatedBranches = set()
2999
3000                if not self.detectBranches:
3001                    if args:
3002                        # start a new branch
3003                        self.initialParent = ""
3004                    else:
3005                        # build on a previous revision
3006                        self.initialParent = parseRevision(self.branch)
3007
3008                self.importChanges(changes)
3009
3010                if not self.silent:
3011                    print ""
3012                    if len(self.updatedBranches) > 0:
3013                        sys.stdout.write("Updated branches: ")
3014                        for b in self.updatedBranches:
3015                            sys.stdout.write("%s " % b)
3016                        sys.stdout.write("\n")
3017
3018        if gitConfigBool("git-p4.importLabels"):
3019            self.importLabels = True
3020
3021        if self.importLabels:
3022            p4Labels = getP4Labels(self.depotPaths)
3023            gitTags = getGitTags()
3024
3025            missingP4Labels = p4Labels - gitTags
3026            self.importP4Labels(self.gitStream, missingP4Labels)
3027
3028        self.gitStream.close()
3029        if self.importProcess.wait() != 0:
3030            die("fast-import failed: %s" % self.gitError.read())
3031        self.gitOutput.close()
3032        self.gitError.close()
3033
3034        # Cleanup temporary branches created during import
3035        if self.tempBranches != []:
3036            for branch in self.tempBranches:
3037                read_pipe("git update-ref -d %s" % branch)
3038            os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3039
3040        # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3041        # a convenient shortcut refname "p4".
3042        if self.importIntoRemotes:
3043            head_ref = self.refPrefix + "HEAD"
3044            if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3045                system(["git", "symbolic-ref", head_ref, self.branch])
3046
3047        return True
3048
3049class P4Rebase(Command):
3050    def __init__(self):
3051        Command.__init__(self)
3052        self.options = [
3053                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3054        ]
3055        self.importLabels = False
3056        self.description = ("Fetches the latest revision from perforce and "
3057                            + "rebases the current work (branch) against it")
3058
3059    def run(self, args):
3060        sync = P4Sync()
3061        sync.importLabels = self.importLabels
3062        sync.run([])
3063
3064        return self.rebase()
3065
3066    def rebase(self):
3067        if os.system("git update-index --refresh") != 0:
3068            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3069        if len(read_pipe("git diff-index HEAD --")) > 0:
3070            die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3071
3072        [upstream, settings] = findUpstreamBranchPoint()
3073        if len(upstream) == 0:
3074            die("Cannot find upstream branchpoint for rebase")
3075
3076        # the branchpoint may be p4/foo~3, so strip off the parent
3077        upstream = re.sub("~[0-9]+$", "", upstream)
3078
3079        print "Rebasing the current branch onto %s" % upstream
3080        oldHead = read_pipe("git rev-parse HEAD").strip()
3081        system("git rebase %s" % upstream)
3082        system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
3083        return True
3084
3085class P4Clone(P4Sync):
3086    def __init__(self):
3087        P4Sync.__init__(self)
3088        self.description = "Creates a new git repository and imports from Perforce into it"
3089        self.usage = "usage: %prog [options] //depot/path[@revRange]"
3090        self.options += [
3091            optparse.make_option("--destination", dest="cloneDestination",
3092                                 action='store', default=None,
3093                                 help="where to leave result of the clone"),
3094            optparse.make_option("-/", dest="cloneExclude",
3095                                 action="append", type="string",
3096                                 help="exclude depot path"),
3097            optparse.make_option("--bare", dest="cloneBare",
3098                                 action="store_true", default=False),
3099        ]
3100        self.cloneDestination = None
3101        self.needsGit = False
3102        self.cloneBare = False
3103
3104    # This is required for the "append" cloneExclude action
3105    def ensure_value(self, attr, value):
3106        if not hasattr(self, attr) or getattr(self, attr) is None:
3107            setattr(self, attr, value)
3108        return getattr(self, attr)
3109
3110    def defaultDestination(self, args):
3111        ## TODO: use common prefix of args?
3112        depotPath = args[0]
3113        depotDir = re.sub("(@[^@]*)$", "", depotPath)
3114        depotDir = re.sub("(#[^#]*)$", "", depotDir)
3115        depotDir = re.sub(r"\.\.\.$", "", depotDir)
3116        depotDir = re.sub(r"/$", "", depotDir)
3117        return os.path.split(depotDir)[1]
3118
3119    def run(self, args):
3120        if len(args) < 1:
3121            return False
3122
3123        if self.keepRepoPath and not self.cloneDestination:
3124            sys.stderr.write("Must specify destination for --keep-path\n")
3125            sys.exit(1)
3126
3127        depotPaths = args
3128
3129        if not self.cloneDestination and len(depotPaths) > 1:
3130            self.cloneDestination = depotPaths[-1]
3131            depotPaths = depotPaths[:-1]
3132
3133        self.cloneExclude = ["/"+p for p in self.cloneExclude]
3134        for p in depotPaths:
3135            if not p.startswith("//"):
3136                sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3137                return False
3138
3139        if not self.cloneDestination:
3140            self.cloneDestination = self.defaultDestination(args)
3141
3142        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3143
3144        if not os.path.exists(self.cloneDestination):
3145            os.makedirs(self.cloneDestination)
3146        chdir(self.cloneDestination)
3147
3148        init_cmd = [ "git", "init" ]
3149        if self.cloneBare:
3150            init_cmd.append("--bare")
3151        retcode = subprocess.call(init_cmd)
3152        if retcode:
3153            raise CalledProcessError(retcode, init_cmd)
3154
3155        if not P4Sync.run(self, depotPaths):
3156            return False
3157
3158        # create a master branch and check out a work tree
3159        if gitBranchExists(self.branch):
3160            system([ "git", "branch", "master", self.branch ])
3161            if not self.cloneBare:
3162                system([ "git", "checkout", "-f" ])
3163        else:
3164            print 'Not checking out any branch, use ' \
3165                  '"git checkout -q -b master <branch>"'
3166
3167        # auto-set this variable if invoked with --use-client-spec
3168        if self.useClientSpec_from_options:
3169            system("git config --bool git-p4.useclientspec true")
3170
3171        return True
3172
3173class P4Branches(Command):
3174    def __init__(self):
3175        Command.__init__(self)
3176        self.options = [ ]
3177        self.description = ("Shows the git branches that hold imports and their "
3178                            + "corresponding perforce depot paths")
3179        self.verbose = False
3180
3181    def run(self, args):
3182        if originP4BranchesExist():
3183            createOrUpdateBranchesFromOrigin()
3184
3185        cmdline = "git rev-parse --symbolic "
3186        cmdline += " --remotes"
3187
3188        for line in read_pipe_lines(cmdline):
3189            line = line.strip()
3190
3191            if not line.startswith('p4/') or line == "p4/HEAD":
3192                continue
3193            branch = line
3194
3195            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3196            settings = extractSettingsGitLog(log)
3197
3198            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3199        return True
3200
3201class HelpFormatter(optparse.IndentedHelpFormatter):
3202    def __init__(self):
3203        optparse.IndentedHelpFormatter.__init__(self)
3204
3205    def format_description(self, description):
3206        if description:
3207            return description + "\n"
3208        else:
3209            return ""
3210
3211def printUsage(commands):
3212    print "usage: %s <command> [options]" % sys.argv[0]
3213    print ""
3214    print "valid commands: %s" % ", ".join(commands)
3215    print ""
3216    print "Try %s <command> --help for command specific help." % sys.argv[0]
3217    print ""
3218
3219commands = {
3220    "debug" : P4Debug,
3221    "submit" : P4Submit,
3222    "commit" : P4Submit,
3223    "sync" : P4Sync,
3224    "rebase" : P4Rebase,
3225    "clone" : P4Clone,
3226    "rollback" : P4RollBack,
3227    "branches" : P4Branches
3228}
3229
3230
3231def main():
3232    if len(sys.argv[1:]) == 0:
3233        printUsage(commands.keys())
3234        sys.exit(2)
3235
3236    cmdName = sys.argv[1]
3237    try:
3238        klass = commands[cmdName]
3239        cmd = klass()
3240    except KeyError:
3241        print "unknown command %s" % cmdName
3242        print ""
3243        printUsage(commands.keys())
3244        sys.exit(2)
3245
3246    options = cmd.options
3247    cmd.gitdir = os.environ.get("GIT_DIR", None)
3248
3249    args = sys.argv[2:]
3250
3251    options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3252    if cmd.needsGit:
3253        options.append(optparse.make_option("--git-dir", dest="gitdir"))
3254
3255    parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3256                                   options,
3257                                   description = cmd.description,
3258                                   formatter = HelpFormatter())
3259
3260    (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3261    global verbose
3262    verbose = cmd.verbose
3263    if cmd.needsGit:
3264        if cmd.gitdir == None:
3265            cmd.gitdir = os.path.abspath(".git")
3266            if not isValidGitDir(cmd.gitdir):
3267                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3268                if os.path.exists(cmd.gitdir):
3269                    cdup = read_pipe("git rev-parse --show-cdup").strip()
3270                    if len(cdup) > 0:
3271                        chdir(cdup);
3272
3273        if not isValidGitDir(cmd.gitdir):
3274            if isValidGitDir(cmd.gitdir + "/.git"):
3275                cmd.gitdir += "/.git"
3276            else:
3277                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3278
3279        os.environ["GIT_DIR"] = cmd.gitdir
3280
3281    if not cmd.run(args):
3282        parser.print_help()
3283        sys.exit(2)
3284
3285
3286if __name__ == '__main__':
3287    main()