git-p4.pyon commit Merge branch 'ah/reflog-typofix-in-error' (bc1c600)
   1#!/usr/bin/env python
   2#
   3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4#
   5# Author: Simon Hausmann <simon@lst.de>
   6# Copyright: 2007 Simon Hausmann <simon@lst.de>
   7#            2007 Trolltech ASA
   8# License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9#
  10import sys
  11if sys.hexversion < 0x02040000:
  12    # The limiter is the subprocess module
  13    sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  14    sys.exit(1)
  15import os
  16import optparse
  17import marshal
  18import subprocess
  19import tempfile
  20import time
  21import platform
  22import re
  23import shutil
  24import stat
  25
  26try:
  27    from subprocess import CalledProcessError
  28except ImportError:
  29    # from python2.7:subprocess.py
  30    # Exception classes used by this module.
  31    class CalledProcessError(Exception):
  32        """This exception is raised when a process run by check_call() returns
  33        a non-zero exit status.  The exit status will be stored in the
  34        returncode attribute."""
  35        def __init__(self, returncode, cmd):
  36            self.returncode = returncode
  37            self.cmd = cmd
  38        def __str__(self):
  39            return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
  40
  41verbose = False
  42
  43# Only labels/tags matching this will be imported/exported
  44defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  45
  46# Grab changes in blocks of this many revisions, unless otherwise requested
  47defaultBlockSize = 512
  48
  49def p4_build_cmd(cmd):
  50    """Build a suitable p4 command line.
  51
  52    This consolidates building and returning a p4 command line into one
  53    location. It means that hooking into the environment, or other configuration
  54    can be done more easily.
  55    """
  56    real_cmd = ["p4"]
  57
  58    user = gitConfig("git-p4.user")
  59    if len(user) > 0:
  60        real_cmd += ["-u",user]
  61
  62    password = gitConfig("git-p4.password")
  63    if len(password) > 0:
  64        real_cmd += ["-P", password]
  65
  66    port = gitConfig("git-p4.port")
  67    if len(port) > 0:
  68        real_cmd += ["-p", port]
  69
  70    host = gitConfig("git-p4.host")
  71    if len(host) > 0:
  72        real_cmd += ["-H", host]
  73
  74    client = gitConfig("git-p4.client")
  75    if len(client) > 0:
  76        real_cmd += ["-c", client]
  77
  78
  79    if isinstance(cmd,basestring):
  80        real_cmd = ' '.join(real_cmd) + ' ' + cmd
  81    else:
  82        real_cmd += cmd
  83    return real_cmd
  84
  85def chdir(path, is_client_path=False):
  86    """Do chdir to the given path, and set the PWD environment
  87       variable for use by P4.  It does not look at getcwd() output.
  88       Since we're not using the shell, it is necessary to set the
  89       PWD environment variable explicitly.
  90
  91       Normally, expand the path to force it to be absolute.  This
  92       addresses the use of relative path names inside P4 settings,
  93       e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
  94       as given; it looks for .p4config using PWD.
  95
  96       If is_client_path, the path was handed to us directly by p4,
  97       and may be a symbolic link.  Do not call os.getcwd() in this
  98       case, because it will cause p4 to think that PWD is not inside
  99       the client path.
 100       """
 101
 102    os.chdir(path)
 103    if not is_client_path:
 104        path = os.getcwd()
 105    os.environ['PWD'] = path
 106
 107def die(msg):
 108    if verbose:
 109        raise Exception(msg)
 110    else:
 111        sys.stderr.write(msg + "\n")
 112        sys.exit(1)
 113
 114def write_pipe(c, stdin):
 115    if verbose:
 116        sys.stderr.write('Writing pipe: %s\n' % str(c))
 117
 118    expand = isinstance(c,basestring)
 119    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
 120    pipe = p.stdin
 121    val = pipe.write(stdin)
 122    pipe.close()
 123    if p.wait():
 124        die('Command failed: %s' % str(c))
 125
 126    return val
 127
 128def p4_write_pipe(c, stdin):
 129    real_cmd = p4_build_cmd(c)
 130    return write_pipe(real_cmd, stdin)
 131
 132def read_pipe(c, ignore_error=False):
 133    if verbose:
 134        sys.stderr.write('Reading pipe: %s\n' % str(c))
 135
 136    expand = isinstance(c,basestring)
 137    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 138    pipe = p.stdout
 139    val = pipe.read()
 140    if p.wait() and not ignore_error:
 141        die('Command failed: %s' % str(c))
 142
 143    return val
 144
 145def p4_read_pipe(c, ignore_error=False):
 146    real_cmd = p4_build_cmd(c)
 147    return read_pipe(real_cmd, ignore_error)
 148
 149def read_pipe_lines(c):
 150    if verbose:
 151        sys.stderr.write('Reading pipe: %s\n' % str(c))
 152
 153    expand = isinstance(c, basestring)
 154    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 155    pipe = p.stdout
 156    val = pipe.readlines()
 157    if pipe.close() or p.wait():
 158        die('Command failed: %s' % str(c))
 159
 160    return val
 161
 162def p4_read_pipe_lines(c):
 163    """Specifically invoke p4 on the command supplied. """
 164    real_cmd = p4_build_cmd(c)
 165    return read_pipe_lines(real_cmd)
 166
 167def p4_has_command(cmd):
 168    """Ask p4 for help on this command.  If it returns an error, the
 169       command does not exist in this version of p4."""
 170    real_cmd = p4_build_cmd(["help", cmd])
 171    p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 172                                   stderr=subprocess.PIPE)
 173    p.communicate()
 174    return p.returncode == 0
 175
 176def p4_has_move_command():
 177    """See if the move command exists, that it supports -k, and that
 178       it has not been administratively disabled.  The arguments
 179       must be correct, but the filenames do not have to exist.  Use
 180       ones with wildcards so even if they exist, it will fail."""
 181
 182    if not p4_has_command("move"):
 183        return False
 184    cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 185    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 186    (out, err) = p.communicate()
 187    # return code will be 1 in either case
 188    if err.find("Invalid option") >= 0:
 189        return False
 190    if err.find("disabled") >= 0:
 191        return False
 192    # assume it failed because @... was invalid changelist
 193    return True
 194
 195def system(cmd):
 196    expand = isinstance(cmd,basestring)
 197    if verbose:
 198        sys.stderr.write("executing %s\n" % str(cmd))
 199    retcode = subprocess.call(cmd, shell=expand)
 200    if retcode:
 201        raise CalledProcessError(retcode, cmd)
 202
 203def p4_system(cmd):
 204    """Specifically invoke p4 as the system command. """
 205    real_cmd = p4_build_cmd(cmd)
 206    expand = isinstance(real_cmd, basestring)
 207    retcode = subprocess.call(real_cmd, shell=expand)
 208    if retcode:
 209        raise CalledProcessError(retcode, real_cmd)
 210
 211_p4_version_string = None
 212def p4_version_string():
 213    """Read the version string, showing just the last line, which
 214       hopefully is the interesting version bit.
 215
 216       $ p4 -V
 217       Perforce - The Fast Software Configuration Management System.
 218       Copyright 1995-2011 Perforce Software.  All rights reserved.
 219       Rev. P4/NTX86/2011.1/393975 (2011/12/16).
 220    """
 221    global _p4_version_string
 222    if not _p4_version_string:
 223        a = p4_read_pipe_lines(["-V"])
 224        _p4_version_string = a[-1].rstrip()
 225    return _p4_version_string
 226
 227def p4_integrate(src, dest):
 228    p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 229
 230def p4_sync(f, *options):
 231    p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 232
 233def p4_add(f):
 234    # forcibly add file names with wildcards
 235    if wildcard_present(f):
 236        p4_system(["add", "-f", f])
 237    else:
 238        p4_system(["add", f])
 239
 240def p4_delete(f):
 241    p4_system(["delete", wildcard_encode(f)])
 242
 243def p4_edit(f):
 244    p4_system(["edit", wildcard_encode(f)])
 245
 246def p4_revert(f):
 247    p4_system(["revert", wildcard_encode(f)])
 248
 249def p4_reopen(type, f):
 250    p4_system(["reopen", "-t", type, wildcard_encode(f)])
 251
 252def p4_move(src, dest):
 253    p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 254
 255def p4_last_change():
 256    results = p4CmdList(["changes", "-m", "1"])
 257    return int(results[0]['change'])
 258
 259def p4_describe(change):
 260    """Make sure it returns a valid result by checking for
 261       the presence of field "time".  Return a dict of the
 262       results."""
 263
 264    ds = p4CmdList(["describe", "-s", str(change)])
 265    if len(ds) != 1:
 266        die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 267
 268    d = ds[0]
 269
 270    if "p4ExitCode" in d:
 271        die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 272                                                      str(d)))
 273    if "code" in d:
 274        if d["code"] == "error":
 275            die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 276
 277    if "time" not in d:
 278        die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 279
 280    return d
 281
 282#
 283# Canonicalize the p4 type and return a tuple of the
 284# base type, plus any modifiers.  See "p4 help filetypes"
 285# for a list and explanation.
 286#
 287def split_p4_type(p4type):
 288
 289    p4_filetypes_historical = {
 290        "ctempobj": "binary+Sw",
 291        "ctext": "text+C",
 292        "cxtext": "text+Cx",
 293        "ktext": "text+k",
 294        "kxtext": "text+kx",
 295        "ltext": "text+F",
 296        "tempobj": "binary+FSw",
 297        "ubinary": "binary+F",
 298        "uresource": "resource+F",
 299        "uxbinary": "binary+Fx",
 300        "xbinary": "binary+x",
 301        "xltext": "text+Fx",
 302        "xtempobj": "binary+Swx",
 303        "xtext": "text+x",
 304        "xunicode": "unicode+x",
 305        "xutf16": "utf16+x",
 306    }
 307    if p4type in p4_filetypes_historical:
 308        p4type = p4_filetypes_historical[p4type]
 309    mods = ""
 310    s = p4type.split("+")
 311    base = s[0]
 312    mods = ""
 313    if len(s) > 1:
 314        mods = s[1]
 315    return (base, mods)
 316
 317#
 318# return the raw p4 type of a file (text, text+ko, etc)
 319#
 320def p4_type(f):
 321    results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
 322    return results[0]['headType']
 323
 324#
 325# Given a type base and modifier, return a regexp matching
 326# the keywords that can be expanded in the file
 327#
 328def p4_keywords_regexp_for_type(base, type_mods):
 329    if base in ("text", "unicode", "binary"):
 330        kwords = None
 331        if "ko" in type_mods:
 332            kwords = 'Id|Header'
 333        elif "k" in type_mods:
 334            kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 335        else:
 336            return None
 337        pattern = r"""
 338            \$              # Starts with a dollar, followed by...
 339            (%s)            # one of the keywords, followed by...
 340            (:[^$\n]+)?     # possibly an old expansion, followed by...
 341            \$              # another dollar
 342            """ % kwords
 343        return pattern
 344    else:
 345        return None
 346
 347#
 348# Given a file, return a regexp matching the possible
 349# RCS keywords that will be expanded, or None for files
 350# with kw expansion turned off.
 351#
 352def p4_keywords_regexp_for_file(file):
 353    if not os.path.exists(file):
 354        return None
 355    else:
 356        (type_base, type_mods) = split_p4_type(p4_type(file))
 357        return p4_keywords_regexp_for_type(type_base, type_mods)
 358
 359def setP4ExecBit(file, mode):
 360    # Reopens an already open file and changes the execute bit to match
 361    # the execute bit setting in the passed in mode.
 362
 363    p4Type = "+x"
 364
 365    if not isModeExec(mode):
 366        p4Type = getP4OpenedType(file)
 367        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 368        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 369        if p4Type[-1] == "+":
 370            p4Type = p4Type[0:-1]
 371
 372    p4_reopen(p4Type, file)
 373
 374def getP4OpenedType(file):
 375    # Returns the perforce file type for the given file.
 376
 377    result = p4_read_pipe(["opened", wildcard_encode(file)])
 378    match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
 379    if match:
 380        return match.group(1)
 381    else:
 382        die("Could not determine file type for %s (result: '%s')" % (file, result))
 383
 384# Return the set of all p4 labels
 385def getP4Labels(depotPaths):
 386    labels = set()
 387    if isinstance(depotPaths,basestring):
 388        depotPaths = [depotPaths]
 389
 390    for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 391        label = l['label']
 392        labels.add(label)
 393
 394    return labels
 395
 396# Return the set of all git tags
 397def getGitTags():
 398    gitTags = set()
 399    for line in read_pipe_lines(["git", "tag"]):
 400        tag = line.strip()
 401        gitTags.add(tag)
 402    return gitTags
 403
 404def diffTreePattern():
 405    # This is a simple generator for the diff tree regex pattern. This could be
 406    # a class variable if this and parseDiffTreeEntry were a part of a class.
 407    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 408    while True:
 409        yield pattern
 410
 411def parseDiffTreeEntry(entry):
 412    """Parses a single diff tree entry into its component elements.
 413
 414    See git-diff-tree(1) manpage for details about the format of the diff
 415    output. This method returns a dictionary with the following elements:
 416
 417    src_mode - The mode of the source file
 418    dst_mode - The mode of the destination file
 419    src_sha1 - The sha1 for the source file
 420    dst_sha1 - The sha1 fr the destination file
 421    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 422    status_score - The score for the status (applicable for 'C' and 'R'
 423                   statuses). This is None if there is no score.
 424    src - The path for the source file.
 425    dst - The path for the destination file. This is only present for
 426          copy or renames. If it is not present, this is None.
 427
 428    If the pattern is not matched, None is returned."""
 429
 430    match = diffTreePattern().next().match(entry)
 431    if match:
 432        return {
 433            'src_mode': match.group(1),
 434            'dst_mode': match.group(2),
 435            'src_sha1': match.group(3),
 436            'dst_sha1': match.group(4),
 437            'status': match.group(5),
 438            'status_score': match.group(6),
 439            'src': match.group(7),
 440            'dst': match.group(10)
 441        }
 442    return None
 443
 444def isModeExec(mode):
 445    # Returns True if the given git mode represents an executable file,
 446    # otherwise False.
 447    return mode[-3:] == "755"
 448
 449def isModeExecChanged(src_mode, dst_mode):
 450    return isModeExec(src_mode) != isModeExec(dst_mode)
 451
 452def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 453
 454    if isinstance(cmd,basestring):
 455        cmd = "-G " + cmd
 456        expand = True
 457    else:
 458        cmd = ["-G"] + cmd
 459        expand = False
 460
 461    cmd = p4_build_cmd(cmd)
 462    if verbose:
 463        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 464
 465    # Use a temporary file to avoid deadlocks without
 466    # subprocess.communicate(), which would put another copy
 467    # of stdout into memory.
 468    stdin_file = None
 469    if stdin is not None:
 470        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 471        if isinstance(stdin,basestring):
 472            stdin_file.write(stdin)
 473        else:
 474            for i in stdin:
 475                stdin_file.write(i + '\n')
 476        stdin_file.flush()
 477        stdin_file.seek(0)
 478
 479    p4 = subprocess.Popen(cmd,
 480                          shell=expand,
 481                          stdin=stdin_file,
 482                          stdout=subprocess.PIPE)
 483
 484    result = []
 485    try:
 486        while True:
 487            entry = marshal.load(p4.stdout)
 488            if cb is not None:
 489                cb(entry)
 490            else:
 491                result.append(entry)
 492    except EOFError:
 493        pass
 494    exitCode = p4.wait()
 495    if exitCode != 0:
 496        entry = {}
 497        entry["p4ExitCode"] = exitCode
 498        result.append(entry)
 499
 500    return result
 501
 502def p4Cmd(cmd):
 503    list = p4CmdList(cmd)
 504    result = {}
 505    for entry in list:
 506        result.update(entry)
 507    return result;
 508
 509def p4Where(depotPath):
 510    if not depotPath.endswith("/"):
 511        depotPath += "/"
 512    depotPathLong = depotPath + "..."
 513    outputList = p4CmdList(["where", depotPathLong])
 514    output = None
 515    for entry in outputList:
 516        if "depotFile" in entry:
 517            # Search for the base client side depot path, as long as it starts with the branch's P4 path.
 518            # The base path always ends with "/...".
 519            if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
 520                output = entry
 521                break
 522        elif "data" in entry:
 523            data = entry.get("data")
 524            space = data.find(" ")
 525            if data[:space] == depotPath:
 526                output = entry
 527                break
 528    if output == None:
 529        return ""
 530    if output["code"] == "error":
 531        return ""
 532    clientPath = ""
 533    if "path" in output:
 534        clientPath = output.get("path")
 535    elif "data" in output:
 536        data = output.get("data")
 537        lastSpace = data.rfind(" ")
 538        clientPath = data[lastSpace + 1:]
 539
 540    if clientPath.endswith("..."):
 541        clientPath = clientPath[:-3]
 542    return clientPath
 543
 544def currentGitBranch():
 545    return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 546
 547def isValidGitDir(path):
 548    if (os.path.exists(path + "/HEAD")
 549        and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 550        return True;
 551    return False
 552
 553def parseRevision(ref):
 554    return read_pipe("git rev-parse %s" % ref).strip()
 555
 556def branchExists(ref):
 557    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 558                     ignore_error=True)
 559    return len(rev) > 0
 560
 561def extractLogMessageFromGitCommit(commit):
 562    logMessage = ""
 563
 564    ## fixme: title is first line of commit, not 1st paragraph.
 565    foundTitle = False
 566    for log in read_pipe_lines("git cat-file commit %s" % commit):
 567       if not foundTitle:
 568           if len(log) == 1:
 569               foundTitle = True
 570           continue
 571
 572       logMessage += log
 573    return logMessage
 574
 575def extractSettingsGitLog(log):
 576    values = {}
 577    for line in log.split("\n"):
 578        line = line.strip()
 579        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 580        if not m:
 581            continue
 582
 583        assignments = m.group(1).split (':')
 584        for a in assignments:
 585            vals = a.split ('=')
 586            key = vals[0].strip()
 587            val = ('='.join (vals[1:])).strip()
 588            if val.endswith ('\"') and val.startswith('"'):
 589                val = val[1:-1]
 590
 591            values[key] = val
 592
 593    paths = values.get("depot-paths")
 594    if not paths:
 595        paths = values.get("depot-path")
 596    if paths:
 597        values['depot-paths'] = paths.split(',')
 598    return values
 599
 600def gitBranchExists(branch):
 601    proc = subprocess.Popen(["git", "rev-parse", branch],
 602                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 603    return proc.wait() == 0;
 604
 605_gitConfig = {}
 606
 607def gitConfig(key):
 608    if not _gitConfig.has_key(key):
 609        cmd = [ "git", "config", key ]
 610        s = read_pipe(cmd, ignore_error=True)
 611        _gitConfig[key] = s.strip()
 612    return _gitConfig[key]
 613
 614def gitConfigBool(key):
 615    """Return a bool, using git config --bool.  It is True only if the
 616       variable is set to true, and False if set to false or not present
 617       in the config."""
 618
 619    if not _gitConfig.has_key(key):
 620        cmd = [ "git", "config", "--bool", key ]
 621        s = read_pipe(cmd, ignore_error=True)
 622        v = s.strip()
 623        _gitConfig[key] = v == "true"
 624    return _gitConfig[key]
 625
 626def gitConfigList(key):
 627    if not _gitConfig.has_key(key):
 628        s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
 629        _gitConfig[key] = s.strip().split(os.linesep)
 630    return _gitConfig[key]
 631
 632def p4BranchesInGit(branchesAreInRemotes=True):
 633    """Find all the branches whose names start with "p4/", looking
 634       in remotes or heads as specified by the argument.  Return
 635       a dictionary of { branch: revision } for each one found.
 636       The branch names are the short names, without any
 637       "p4/" prefix."""
 638
 639    branches = {}
 640
 641    cmdline = "git rev-parse --symbolic "
 642    if branchesAreInRemotes:
 643        cmdline += "--remotes"
 644    else:
 645        cmdline += "--branches"
 646
 647    for line in read_pipe_lines(cmdline):
 648        line = line.strip()
 649
 650        # only import to p4/
 651        if not line.startswith('p4/'):
 652            continue
 653        # special symbolic ref to p4/master
 654        if line == "p4/HEAD":
 655            continue
 656
 657        # strip off p4/ prefix
 658        branch = line[len("p4/"):]
 659
 660        branches[branch] = parseRevision(line)
 661
 662    return branches
 663
 664def branch_exists(branch):
 665    """Make sure that the given ref name really exists."""
 666
 667    cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 668    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 669    out, _ = p.communicate()
 670    if p.returncode:
 671        return False
 672    # expect exactly one line of output: the branch name
 673    return out.rstrip() == branch
 674
 675def findUpstreamBranchPoint(head = "HEAD"):
 676    branches = p4BranchesInGit()
 677    # map from depot-path to branch name
 678    branchByDepotPath = {}
 679    for branch in branches.keys():
 680        tip = branches[branch]
 681        log = extractLogMessageFromGitCommit(tip)
 682        settings = extractSettingsGitLog(log)
 683        if settings.has_key("depot-paths"):
 684            paths = ",".join(settings["depot-paths"])
 685            branchByDepotPath[paths] = "remotes/p4/" + branch
 686
 687    settings = None
 688    parent = 0
 689    while parent < 65535:
 690        commit = head + "~%s" % parent
 691        log = extractLogMessageFromGitCommit(commit)
 692        settings = extractSettingsGitLog(log)
 693        if settings.has_key("depot-paths"):
 694            paths = ",".join(settings["depot-paths"])
 695            if branchByDepotPath.has_key(paths):
 696                return [branchByDepotPath[paths], settings]
 697
 698        parent = parent + 1
 699
 700    return ["", settings]
 701
 702def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 703    if not silent:
 704        print ("Creating/updating branch(es) in %s based on origin branch(es)"
 705               % localRefPrefix)
 706
 707    originPrefix = "origin/p4/"
 708
 709    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 710        line = line.strip()
 711        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 712            continue
 713
 714        headName = line[len(originPrefix):]
 715        remoteHead = localRefPrefix + headName
 716        originHead = line
 717
 718        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 719        if (not original.has_key('depot-paths')
 720            or not original.has_key('change')):
 721            continue
 722
 723        update = False
 724        if not gitBranchExists(remoteHead):
 725            if verbose:
 726                print "creating %s" % remoteHead
 727            update = True
 728        else:
 729            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 730            if settings.has_key('change') > 0:
 731                if settings['depot-paths'] == original['depot-paths']:
 732                    originP4Change = int(original['change'])
 733                    p4Change = int(settings['change'])
 734                    if originP4Change > p4Change:
 735                        print ("%s (%s) is newer than %s (%s). "
 736                               "Updating p4 branch from origin."
 737                               % (originHead, originP4Change,
 738                                  remoteHead, p4Change))
 739                        update = True
 740                else:
 741                    print ("Ignoring: %s was imported from %s while "
 742                           "%s was imported from %s"
 743                           % (originHead, ','.join(original['depot-paths']),
 744                              remoteHead, ','.join(settings['depot-paths'])))
 745
 746        if update:
 747            system("git update-ref %s %s" % (remoteHead, originHead))
 748
 749def originP4BranchesExist():
 750        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 751
 752
 753def p4ParseNumericChangeRange(parts):
 754    changeStart = int(parts[0][1:])
 755    if parts[1] == '#head':
 756        changeEnd = p4_last_change()
 757    else:
 758        changeEnd = int(parts[1])
 759
 760    return (changeStart, changeEnd)
 761
 762def chooseBlockSize(blockSize):
 763    if blockSize:
 764        return blockSize
 765    else:
 766        return defaultBlockSize
 767
 768def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
 769    assert depotPaths
 770
 771    # Parse the change range into start and end. Try to find integer
 772    # revision ranges as these can be broken up into blocks to avoid
 773    # hitting server-side limits (maxrows, maxscanresults). But if
 774    # that doesn't work, fall back to using the raw revision specifier
 775    # strings, without using block mode.
 776
 777    if changeRange is None or changeRange == '':
 778        changeStart = 1
 779        changeEnd = p4_last_change()
 780        block_size = chooseBlockSize(requestedBlockSize)
 781    else:
 782        parts = changeRange.split(',')
 783        assert len(parts) == 2
 784        try:
 785            (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
 786            block_size = chooseBlockSize(requestedBlockSize)
 787        except:
 788            changeStart = parts[0][1:]
 789            changeEnd = parts[1]
 790            if requestedBlockSize:
 791                die("cannot use --changes-block-size with non-numeric revisions")
 792            block_size = None
 793
 794    # Accumulate change numbers in a dictionary to avoid duplicates
 795    changes = {}
 796
 797    for p in depotPaths:
 798        # Retrieve changes a block at a time, to prevent running
 799        # into a MaxResults/MaxScanRows error from the server.
 800
 801        while True:
 802            cmd = ['changes']
 803
 804            if block_size:
 805                end = min(changeEnd, changeStart + block_size)
 806                revisionRange = "%d,%d" % (changeStart, end)
 807            else:
 808                revisionRange = "%s,%s" % (changeStart, changeEnd)
 809
 810            cmd += ["%s...@%s" % (p, revisionRange)]
 811
 812            for line in p4_read_pipe_lines(cmd):
 813                changeNum = int(line.split(" ")[1])
 814                changes[changeNum] = True
 815
 816            if not block_size:
 817                break
 818
 819            if end >= changeEnd:
 820                break
 821
 822            changeStart = end + 1
 823
 824    changelist = changes.keys()
 825    changelist.sort()
 826    return changelist
 827
 828def p4PathStartsWith(path, prefix):
 829    # This method tries to remedy a potential mixed-case issue:
 830    #
 831    # If UserA adds  //depot/DirA/file1
 832    # and UserB adds //depot/dira/file2
 833    #
 834    # we may or may not have a problem. If you have core.ignorecase=true,
 835    # we treat DirA and dira as the same directory
 836    if gitConfigBool("core.ignorecase"):
 837        return path.lower().startswith(prefix.lower())
 838    return path.startswith(prefix)
 839
 840def getClientSpec():
 841    """Look at the p4 client spec, create a View() object that contains
 842       all the mappings, and return it."""
 843
 844    specList = p4CmdList("client -o")
 845    if len(specList) != 1:
 846        die('Output from "client -o" is %d lines, expecting 1' %
 847            len(specList))
 848
 849    # dictionary of all client parameters
 850    entry = specList[0]
 851
 852    # the //client/ name
 853    client_name = entry["Client"]
 854
 855    # just the keys that start with "View"
 856    view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 857
 858    # hold this new View
 859    view = View(client_name)
 860
 861    # append the lines, in order, to the view
 862    for view_num in range(len(view_keys)):
 863        k = "View%d" % view_num
 864        if k not in view_keys:
 865            die("Expected view key %s missing" % k)
 866        view.append(entry[k])
 867
 868    return view
 869
 870def getClientRoot():
 871    """Grab the client directory."""
 872
 873    output = p4CmdList("client -o")
 874    if len(output) != 1:
 875        die('Output from "client -o" is %d lines, expecting 1' % len(output))
 876
 877    entry = output[0]
 878    if "Root" not in entry:
 879        die('Client has no "Root"')
 880
 881    return entry["Root"]
 882
 883#
 884# P4 wildcards are not allowed in filenames.  P4 complains
 885# if you simply add them, but you can force it with "-f", in
 886# which case it translates them into %xx encoding internally.
 887#
 888def wildcard_decode(path):
 889    # Search for and fix just these four characters.  Do % last so
 890    # that fixing it does not inadvertently create new %-escapes.
 891    # Cannot have * in a filename in windows; untested as to
 892    # what p4 would do in such a case.
 893    if not platform.system() == "Windows":
 894        path = path.replace("%2A", "*")
 895    path = path.replace("%23", "#") \
 896               .replace("%40", "@") \
 897               .replace("%25", "%")
 898    return path
 899
 900def wildcard_encode(path):
 901    # do % first to avoid double-encoding the %s introduced here
 902    path = path.replace("%", "%25") \
 903               .replace("*", "%2A") \
 904               .replace("#", "%23") \
 905               .replace("@", "%40")
 906    return path
 907
 908def wildcard_present(path):
 909    m = re.search("[*#@%]", path)
 910    return m is not None
 911
 912class Command:
 913    def __init__(self):
 914        self.usage = "usage: %prog [options]"
 915        self.needsGit = True
 916        self.verbose = False
 917
 918class P4UserMap:
 919    def __init__(self):
 920        self.userMapFromPerforceServer = False
 921        self.myP4UserId = None
 922
 923    def p4UserId(self):
 924        if self.myP4UserId:
 925            return self.myP4UserId
 926
 927        results = p4CmdList("user -o")
 928        for r in results:
 929            if r.has_key('User'):
 930                self.myP4UserId = r['User']
 931                return r['User']
 932        die("Could not find your p4 user id")
 933
 934    def p4UserIsMe(self, p4User):
 935        # return True if the given p4 user is actually me
 936        me = self.p4UserId()
 937        if not p4User or p4User != me:
 938            return False
 939        else:
 940            return True
 941
 942    def getUserCacheFilename(self):
 943        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 944        return home + "/.gitp4-usercache.txt"
 945
 946    def getUserMapFromPerforceServer(self):
 947        if self.userMapFromPerforceServer:
 948            return
 949        self.users = {}
 950        self.emails = {}
 951
 952        for output in p4CmdList("users"):
 953            if not output.has_key("User"):
 954                continue
 955            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 956            self.emails[output["Email"]] = output["User"]
 957
 958
 959        s = ''
 960        for (key, val) in self.users.items():
 961            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 962
 963        open(self.getUserCacheFilename(), "wb").write(s)
 964        self.userMapFromPerforceServer = True
 965
 966    def loadUserMapFromCache(self):
 967        self.users = {}
 968        self.userMapFromPerforceServer = False
 969        try:
 970            cache = open(self.getUserCacheFilename(), "rb")
 971            lines = cache.readlines()
 972            cache.close()
 973            for line in lines:
 974                entry = line.strip().split("\t")
 975                self.users[entry[0]] = entry[1]
 976        except IOError:
 977            self.getUserMapFromPerforceServer()
 978
 979class P4Debug(Command):
 980    def __init__(self):
 981        Command.__init__(self)
 982        self.options = []
 983        self.description = "A tool to debug the output of p4 -G."
 984        self.needsGit = False
 985
 986    def run(self, args):
 987        j = 0
 988        for output in p4CmdList(args):
 989            print 'Element: %d' % j
 990            j += 1
 991            print output
 992        return True
 993
 994class P4RollBack(Command):
 995    def __init__(self):
 996        Command.__init__(self)
 997        self.options = [
 998            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 999        ]
1000        self.description = "A tool to debug the multi-branch import. Don't use :)"
1001        self.rollbackLocalBranches = False
1002
1003    def run(self, args):
1004        if len(args) != 1:
1005            return False
1006        maxChange = int(args[0])
1007
1008        if "p4ExitCode" in p4Cmd("changes -m 1"):
1009            die("Problems executing p4");
1010
1011        if self.rollbackLocalBranches:
1012            refPrefix = "refs/heads/"
1013            lines = read_pipe_lines("git rev-parse --symbolic --branches")
1014        else:
1015            refPrefix = "refs/remotes/"
1016            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1017
1018        for line in lines:
1019            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1020                line = line.strip()
1021                ref = refPrefix + line
1022                log = extractLogMessageFromGitCommit(ref)
1023                settings = extractSettingsGitLog(log)
1024
1025                depotPaths = settings['depot-paths']
1026                change = settings['change']
1027
1028                changed = False
1029
1030                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1031                                                           for p in depotPaths]))) == 0:
1032                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1033                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1034                    continue
1035
1036                while change and int(change) > maxChange:
1037                    changed = True
1038                    if self.verbose:
1039                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1040                    system("git update-ref %s \"%s^\"" % (ref, ref))
1041                    log = extractLogMessageFromGitCommit(ref)
1042                    settings =  extractSettingsGitLog(log)
1043
1044
1045                    depotPaths = settings['depot-paths']
1046                    change = settings['change']
1047
1048                if changed:
1049                    print "%s rewound to %s" % (ref, change)
1050
1051        return True
1052
1053class P4Submit(Command, P4UserMap):
1054
1055    conflict_behavior_choices = ("ask", "skip", "quit")
1056
1057    def __init__(self):
1058        Command.__init__(self)
1059        P4UserMap.__init__(self)
1060        self.options = [
1061                optparse.make_option("--origin", dest="origin"),
1062                optparse.make_option("-M", dest="detectRenames", action="store_true"),
1063                # preserve the user, requires relevant p4 permissions
1064                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1065                optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1066                optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1067                optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1068                optparse.make_option("--conflict", dest="conflict_behavior",
1069                                     choices=self.conflict_behavior_choices),
1070                optparse.make_option("--branch", dest="branch"),
1071        ]
1072        self.description = "Submit changes from git to the perforce depot."
1073        self.usage += " [name of git branch to submit into perforce depot]"
1074        self.origin = ""
1075        self.detectRenames = False
1076        self.preserveUser = gitConfigBool("git-p4.preserveUser")
1077        self.dry_run = False
1078        self.prepare_p4_only = False
1079        self.conflict_behavior = None
1080        self.isWindows = (platform.system() == "Windows")
1081        self.exportLabels = False
1082        self.p4HasMoveCommand = p4_has_move_command()
1083        self.branch = None
1084
1085    def check(self):
1086        if len(p4CmdList("opened ...")) > 0:
1087            die("You have files opened with perforce! Close them before starting the sync.")
1088
1089    def separate_jobs_from_description(self, message):
1090        """Extract and return a possible Jobs field in the commit
1091           message.  It goes into a separate section in the p4 change
1092           specification.
1093
1094           A jobs line starts with "Jobs:" and looks like a new field
1095           in a form.  Values are white-space separated on the same
1096           line or on following lines that start with a tab.
1097
1098           This does not parse and extract the full git commit message
1099           like a p4 form.  It just sees the Jobs: line as a marker
1100           to pass everything from then on directly into the p4 form,
1101           but outside the description section.
1102
1103           Return a tuple (stripped log message, jobs string)."""
1104
1105        m = re.search(r'^Jobs:', message, re.MULTILINE)
1106        if m is None:
1107            return (message, None)
1108
1109        jobtext = message[m.start():]
1110        stripped_message = message[:m.start()].rstrip()
1111        return (stripped_message, jobtext)
1112
1113    def prepareLogMessage(self, template, message, jobs):
1114        """Edits the template returned from "p4 change -o" to insert
1115           the message in the Description field, and the jobs text in
1116           the Jobs field."""
1117        result = ""
1118
1119        inDescriptionSection = False
1120
1121        for line in template.split("\n"):
1122            if line.startswith("#"):
1123                result += line + "\n"
1124                continue
1125
1126            if inDescriptionSection:
1127                if line.startswith("Files:") or line.startswith("Jobs:"):
1128                    inDescriptionSection = False
1129                    # insert Jobs section
1130                    if jobs:
1131                        result += jobs + "\n"
1132                else:
1133                    continue
1134            else:
1135                if line.startswith("Description:"):
1136                    inDescriptionSection = True
1137                    line += "\n"
1138                    for messageLine in message.split("\n"):
1139                        line += "\t" + messageLine + "\n"
1140
1141            result += line + "\n"
1142
1143        return result
1144
1145    def patchRCSKeywords(self, file, pattern):
1146        # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1147        (handle, outFileName) = tempfile.mkstemp(dir='.')
1148        try:
1149            outFile = os.fdopen(handle, "w+")
1150            inFile = open(file, "r")
1151            regexp = re.compile(pattern, re.VERBOSE)
1152            for line in inFile.readlines():
1153                line = regexp.sub(r'$\1$', line)
1154                outFile.write(line)
1155            inFile.close()
1156            outFile.close()
1157            # Forcibly overwrite the original file
1158            os.unlink(file)
1159            shutil.move(outFileName, file)
1160        except:
1161            # cleanup our temporary file
1162            os.unlink(outFileName)
1163            print "Failed to strip RCS keywords in %s" % file
1164            raise
1165
1166        print "Patched up RCS keywords in %s" % file
1167
1168    def p4UserForCommit(self,id):
1169        # Return the tuple (perforce user,git email) for a given git commit id
1170        self.getUserMapFromPerforceServer()
1171        gitEmail = read_pipe(["git", "log", "--max-count=1",
1172                              "--format=%ae", id])
1173        gitEmail = gitEmail.strip()
1174        if not self.emails.has_key(gitEmail):
1175            return (None,gitEmail)
1176        else:
1177            return (self.emails[gitEmail],gitEmail)
1178
1179    def checkValidP4Users(self,commits):
1180        # check if any git authors cannot be mapped to p4 users
1181        for id in commits:
1182            (user,email) = self.p4UserForCommit(id)
1183            if not user:
1184                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1185                if gitConfigBool("git-p4.allowMissingP4Users"):
1186                    print "%s" % msg
1187                else:
1188                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1189
1190    def lastP4Changelist(self):
1191        # Get back the last changelist number submitted in this client spec. This
1192        # then gets used to patch up the username in the change. If the same
1193        # client spec is being used by multiple processes then this might go
1194        # wrong.
1195        results = p4CmdList("client -o")        # find the current client
1196        client = None
1197        for r in results:
1198            if r.has_key('Client'):
1199                client = r['Client']
1200                break
1201        if not client:
1202            die("could not get client spec")
1203        results = p4CmdList(["changes", "-c", client, "-m", "1"])
1204        for r in results:
1205            if r.has_key('change'):
1206                return r['change']
1207        die("Could not get changelist number for last submit - cannot patch up user details")
1208
1209    def modifyChangelistUser(self, changelist, newUser):
1210        # fixup the user field of a changelist after it has been submitted.
1211        changes = p4CmdList("change -o %s" % changelist)
1212        if len(changes) != 1:
1213            die("Bad output from p4 change modifying %s to user %s" %
1214                (changelist, newUser))
1215
1216        c = changes[0]
1217        if c['User'] == newUser: return   # nothing to do
1218        c['User'] = newUser
1219        input = marshal.dumps(c)
1220
1221        result = p4CmdList("change -f -i", stdin=input)
1222        for r in result:
1223            if r.has_key('code'):
1224                if r['code'] == 'error':
1225                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1226            if r.has_key('data'):
1227                print("Updated user field for changelist %s to %s" % (changelist, newUser))
1228                return
1229        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1230
1231    def canChangeChangelists(self):
1232        # check to see if we have p4 admin or super-user permissions, either of
1233        # which are required to modify changelists.
1234        results = p4CmdList(["protects", self.depotPath])
1235        for r in results:
1236            if r.has_key('perm'):
1237                if r['perm'] == 'admin':
1238                    return 1
1239                if r['perm'] == 'super':
1240                    return 1
1241        return 0
1242
1243    def prepareSubmitTemplate(self):
1244        """Run "p4 change -o" to grab a change specification template.
1245           This does not use "p4 -G", as it is nice to keep the submission
1246           template in original order, since a human might edit it.
1247
1248           Remove lines in the Files section that show changes to files
1249           outside the depot path we're committing into."""
1250
1251        template = ""
1252        inFilesSection = False
1253        for line in p4_read_pipe_lines(['change', '-o']):
1254            if line.endswith("\r\n"):
1255                line = line[:-2] + "\n"
1256            if inFilesSection:
1257                if line.startswith("\t"):
1258                    # path starts and ends with a tab
1259                    path = line[1:]
1260                    lastTab = path.rfind("\t")
1261                    if lastTab != -1:
1262                        path = path[:lastTab]
1263                        if not p4PathStartsWith(path, self.depotPath):
1264                            continue
1265                else:
1266                    inFilesSection = False
1267            else:
1268                if line.startswith("Files:"):
1269                    inFilesSection = True
1270
1271            template += line
1272
1273        return template
1274
1275    def edit_template(self, template_file):
1276        """Invoke the editor to let the user change the submission
1277           message.  Return true if okay to continue with the submit."""
1278
1279        # if configured to skip the editing part, just submit
1280        if gitConfigBool("git-p4.skipSubmitEdit"):
1281            return True
1282
1283        # look at the modification time, to check later if the user saved
1284        # the file
1285        mtime = os.stat(template_file).st_mtime
1286
1287        # invoke the editor
1288        if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1289            editor = os.environ.get("P4EDITOR")
1290        else:
1291            editor = read_pipe("git var GIT_EDITOR").strip()
1292        system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1293
1294        # If the file was not saved, prompt to see if this patch should
1295        # be skipped.  But skip this verification step if configured so.
1296        if gitConfigBool("git-p4.skipSubmitEditCheck"):
1297            return True
1298
1299        # modification time updated means user saved the file
1300        if os.stat(template_file).st_mtime > mtime:
1301            return True
1302
1303        while True:
1304            response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1305            if response == 'y':
1306                return True
1307            if response == 'n':
1308                return False
1309
1310    def get_diff_description(self, editedFiles, filesToAdd):
1311        # diff
1312        if os.environ.has_key("P4DIFF"):
1313            del(os.environ["P4DIFF"])
1314        diff = ""
1315        for editedFile in editedFiles:
1316            diff += p4_read_pipe(['diff', '-du',
1317                                  wildcard_encode(editedFile)])
1318
1319        # new file diff
1320        newdiff = ""
1321        for newFile in filesToAdd:
1322            newdiff += "==== new file ====\n"
1323            newdiff += "--- /dev/null\n"
1324            newdiff += "+++ %s\n" % newFile
1325            f = open(newFile, "r")
1326            for line in f.readlines():
1327                newdiff += "+" + line
1328            f.close()
1329
1330        return (diff + newdiff).replace('\r\n', '\n')
1331
1332    def applyCommit(self, id):
1333        """Apply one commit, return True if it succeeded."""
1334
1335        print "Applying", read_pipe(["git", "show", "-s",
1336                                     "--format=format:%h %s", id])
1337
1338        (p4User, gitEmail) = self.p4UserForCommit(id)
1339
1340        diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1341        filesToAdd = set()
1342        filesToDelete = set()
1343        editedFiles = set()
1344        pureRenameCopy = set()
1345        filesToChangeExecBit = {}
1346
1347        for line in diff:
1348            diff = parseDiffTreeEntry(line)
1349            modifier = diff['status']
1350            path = diff['src']
1351            if modifier == "M":
1352                p4_edit(path)
1353                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1354                    filesToChangeExecBit[path] = diff['dst_mode']
1355                editedFiles.add(path)
1356            elif modifier == "A":
1357                filesToAdd.add(path)
1358                filesToChangeExecBit[path] = diff['dst_mode']
1359                if path in filesToDelete:
1360                    filesToDelete.remove(path)
1361            elif modifier == "D":
1362                filesToDelete.add(path)
1363                if path in filesToAdd:
1364                    filesToAdd.remove(path)
1365            elif modifier == "C":
1366                src, dest = diff['src'], diff['dst']
1367                p4_integrate(src, dest)
1368                pureRenameCopy.add(dest)
1369                if diff['src_sha1'] != diff['dst_sha1']:
1370                    p4_edit(dest)
1371                    pureRenameCopy.discard(dest)
1372                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1373                    p4_edit(dest)
1374                    pureRenameCopy.discard(dest)
1375                    filesToChangeExecBit[dest] = diff['dst_mode']
1376                if self.isWindows:
1377                    # turn off read-only attribute
1378                    os.chmod(dest, stat.S_IWRITE)
1379                os.unlink(dest)
1380                editedFiles.add(dest)
1381            elif modifier == "R":
1382                src, dest = diff['src'], diff['dst']
1383                if self.p4HasMoveCommand:
1384                    p4_edit(src)        # src must be open before move
1385                    p4_move(src, dest)  # opens for (move/delete, move/add)
1386                else:
1387                    p4_integrate(src, dest)
1388                    if diff['src_sha1'] != diff['dst_sha1']:
1389                        p4_edit(dest)
1390                    else:
1391                        pureRenameCopy.add(dest)
1392                if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1393                    if not self.p4HasMoveCommand:
1394                        p4_edit(dest)   # with move: already open, writable
1395                    filesToChangeExecBit[dest] = diff['dst_mode']
1396                if not self.p4HasMoveCommand:
1397                    if self.isWindows:
1398                        os.chmod(dest, stat.S_IWRITE)
1399                    os.unlink(dest)
1400                    filesToDelete.add(src)
1401                editedFiles.add(dest)
1402            else:
1403                die("unknown modifier %s for %s" % (modifier, path))
1404
1405        diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1406        patchcmd = diffcmd + " | git apply "
1407        tryPatchCmd = patchcmd + "--check -"
1408        applyPatchCmd = patchcmd + "--check --apply -"
1409        patch_succeeded = True
1410
1411        if os.system(tryPatchCmd) != 0:
1412            fixed_rcs_keywords = False
1413            patch_succeeded = False
1414            print "Unfortunately applying the change failed!"
1415
1416            # Patch failed, maybe it's just RCS keyword woes. Look through
1417            # the patch to see if that's possible.
1418            if gitConfigBool("git-p4.attemptRCSCleanup"):
1419                file = None
1420                pattern = None
1421                kwfiles = {}
1422                for file in editedFiles | filesToDelete:
1423                    # did this file's delta contain RCS keywords?
1424                    pattern = p4_keywords_regexp_for_file(file)
1425
1426                    if pattern:
1427                        # this file is a possibility...look for RCS keywords.
1428                        regexp = re.compile(pattern, re.VERBOSE)
1429                        for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1430                            if regexp.search(line):
1431                                if verbose:
1432                                    print "got keyword match on %s in %s in %s" % (pattern, line, file)
1433                                kwfiles[file] = pattern
1434                                break
1435
1436                for file in kwfiles:
1437                    if verbose:
1438                        print "zapping %s with %s" % (line,pattern)
1439                    # File is being deleted, so not open in p4.  Must
1440                    # disable the read-only bit on windows.
1441                    if self.isWindows and file not in editedFiles:
1442                        os.chmod(file, stat.S_IWRITE)
1443                    self.patchRCSKeywords(file, kwfiles[file])
1444                    fixed_rcs_keywords = True
1445
1446            if fixed_rcs_keywords:
1447                print "Retrying the patch with RCS keywords cleaned up"
1448                if os.system(tryPatchCmd) == 0:
1449                    patch_succeeded = True
1450
1451        if not patch_succeeded:
1452            for f in editedFiles:
1453                p4_revert(f)
1454            return False
1455
1456        #
1457        # Apply the patch for real, and do add/delete/+x handling.
1458        #
1459        system(applyPatchCmd)
1460
1461        for f in filesToAdd:
1462            p4_add(f)
1463        for f in filesToDelete:
1464            p4_revert(f)
1465            p4_delete(f)
1466
1467        # Set/clear executable bits
1468        for f in filesToChangeExecBit.keys():
1469            mode = filesToChangeExecBit[f]
1470            setP4ExecBit(f, mode)
1471
1472        #
1473        # Build p4 change description, starting with the contents
1474        # of the git commit message.
1475        #
1476        logMessage = extractLogMessageFromGitCommit(id)
1477        logMessage = logMessage.strip()
1478        (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1479
1480        template = self.prepareSubmitTemplate()
1481        submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1482
1483        if self.preserveUser:
1484           submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1485
1486        if self.checkAuthorship and not self.p4UserIsMe(p4User):
1487            submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1488            submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1489            submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1490
1491        separatorLine = "######## everything below this line is just the diff #######\n"
1492        if not self.prepare_p4_only:
1493            submitTemplate += separatorLine
1494            submitTemplate += self.get_diff_description(editedFiles, filesToAdd)
1495
1496        (handle, fileName) = tempfile.mkstemp()
1497        tmpFile = os.fdopen(handle, "w+b")
1498        if self.isWindows:
1499            submitTemplate = submitTemplate.replace("\n", "\r\n")
1500        tmpFile.write(submitTemplate)
1501        tmpFile.close()
1502
1503        if self.prepare_p4_only:
1504            #
1505            # Leave the p4 tree prepared, and the submit template around
1506            # and let the user decide what to do next
1507            #
1508            print
1509            print "P4 workspace prepared for submission."
1510            print "To submit or revert, go to client workspace"
1511            print "  " + self.clientPath
1512            print
1513            print "To submit, use \"p4 submit\" to write a new description,"
1514            print "or \"p4 submit -i <%s\" to use the one prepared by" \
1515                  " \"git p4\"." % fileName
1516            print "You can delete the file \"%s\" when finished." % fileName
1517
1518            if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1519                print "To preserve change ownership by user %s, you must\n" \
1520                      "do \"p4 change -f <change>\" after submitting and\n" \
1521                      "edit the User field."
1522            if pureRenameCopy:
1523                print "After submitting, renamed files must be re-synced."
1524                print "Invoke \"p4 sync -f\" on each of these files:"
1525                for f in pureRenameCopy:
1526                    print "  " + f
1527
1528            print
1529            print "To revert the changes, use \"p4 revert ...\", and delete"
1530            print "the submit template file \"%s\"" % fileName
1531            if filesToAdd:
1532                print "Since the commit adds new files, they must be deleted:"
1533                for f in filesToAdd:
1534                    print "  " + f
1535            print
1536            return True
1537
1538        #
1539        # Let the user edit the change description, then submit it.
1540        #
1541        if self.edit_template(fileName):
1542            # read the edited message and submit
1543            ret = True
1544            tmpFile = open(fileName, "rb")
1545            message = tmpFile.read()
1546            tmpFile.close()
1547            if self.isWindows:
1548                message = message.replace("\r\n", "\n")
1549            submitTemplate = message[:message.index(separatorLine)]
1550            p4_write_pipe(['submit', '-i'], submitTemplate)
1551
1552            if self.preserveUser:
1553                if p4User:
1554                    # Get last changelist number. Cannot easily get it from
1555                    # the submit command output as the output is
1556                    # unmarshalled.
1557                    changelist = self.lastP4Changelist()
1558                    self.modifyChangelistUser(changelist, p4User)
1559
1560            # The rename/copy happened by applying a patch that created a
1561            # new file.  This leaves it writable, which confuses p4.
1562            for f in pureRenameCopy:
1563                p4_sync(f, "-f")
1564
1565        else:
1566            # skip this patch
1567            ret = False
1568            print "Submission cancelled, undoing p4 changes."
1569            for f in editedFiles:
1570                p4_revert(f)
1571            for f in filesToAdd:
1572                p4_revert(f)
1573                os.remove(f)
1574            for f in filesToDelete:
1575                p4_revert(f)
1576
1577        os.remove(fileName)
1578        return ret
1579
1580    # Export git tags as p4 labels. Create a p4 label and then tag
1581    # with that.
1582    def exportGitTags(self, gitTags):
1583        validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1584        if len(validLabelRegexp) == 0:
1585            validLabelRegexp = defaultLabelRegexp
1586        m = re.compile(validLabelRegexp)
1587
1588        for name in gitTags:
1589
1590            if not m.match(name):
1591                if verbose:
1592                    print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1593                continue
1594
1595            # Get the p4 commit this corresponds to
1596            logMessage = extractLogMessageFromGitCommit(name)
1597            values = extractSettingsGitLog(logMessage)
1598
1599            if not values.has_key('change'):
1600                # a tag pointing to something not sent to p4; ignore
1601                if verbose:
1602                    print "git tag %s does not give a p4 commit" % name
1603                continue
1604            else:
1605                changelist = values['change']
1606
1607            # Get the tag details.
1608            inHeader = True
1609            isAnnotated = False
1610            body = []
1611            for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1612                l = l.strip()
1613                if inHeader:
1614                    if re.match(r'tag\s+', l):
1615                        isAnnotated = True
1616                    elif re.match(r'\s*$', l):
1617                        inHeader = False
1618                        continue
1619                else:
1620                    body.append(l)
1621
1622            if not isAnnotated:
1623                body = ["lightweight tag imported by git p4\n"]
1624
1625            # Create the label - use the same view as the client spec we are using
1626            clientSpec = getClientSpec()
1627
1628            labelTemplate  = "Label: %s\n" % name
1629            labelTemplate += "Description:\n"
1630            for b in body:
1631                labelTemplate += "\t" + b + "\n"
1632            labelTemplate += "View:\n"
1633            for depot_side in clientSpec.mappings:
1634                labelTemplate += "\t%s\n" % depot_side
1635
1636            if self.dry_run:
1637                print "Would create p4 label %s for tag" % name
1638            elif self.prepare_p4_only:
1639                print "Not creating p4 label %s for tag due to option" \
1640                      " --prepare-p4-only" % name
1641            else:
1642                p4_write_pipe(["label", "-i"], labelTemplate)
1643
1644                # Use the label
1645                p4_system(["tag", "-l", name] +
1646                          ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1647
1648                if verbose:
1649                    print "created p4 label for tag %s" % name
1650
1651    def run(self, args):
1652        if len(args) == 0:
1653            self.master = currentGitBranch()
1654            if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1655                die("Detecting current git branch failed!")
1656        elif len(args) == 1:
1657            self.master = args[0]
1658            if not branchExists(self.master):
1659                die("Branch %s does not exist" % self.master)
1660        else:
1661            return False
1662
1663        allowSubmit = gitConfig("git-p4.allowSubmit")
1664        if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1665            die("%s is not in git-p4.allowSubmit" % self.master)
1666
1667        [upstream, settings] = findUpstreamBranchPoint()
1668        self.depotPath = settings['depot-paths'][0]
1669        if len(self.origin) == 0:
1670            self.origin = upstream
1671
1672        if self.preserveUser:
1673            if not self.canChangeChangelists():
1674                die("Cannot preserve user names without p4 super-user or admin permissions")
1675
1676        # if not set from the command line, try the config file
1677        if self.conflict_behavior is None:
1678            val = gitConfig("git-p4.conflict")
1679            if val:
1680                if val not in self.conflict_behavior_choices:
1681                    die("Invalid value '%s' for config git-p4.conflict" % val)
1682            else:
1683                val = "ask"
1684            self.conflict_behavior = val
1685
1686        if self.verbose:
1687            print "Origin branch is " + self.origin
1688
1689        if len(self.depotPath) == 0:
1690            print "Internal error: cannot locate perforce depot path from existing branches"
1691            sys.exit(128)
1692
1693        self.useClientSpec = False
1694        if gitConfigBool("git-p4.useclientspec"):
1695            self.useClientSpec = True
1696        if self.useClientSpec:
1697            self.clientSpecDirs = getClientSpec()
1698
1699        # Check for the existance of P4 branches
1700        branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1701
1702        if self.useClientSpec and not branchesDetected:
1703            # all files are relative to the client spec
1704            self.clientPath = getClientRoot()
1705        else:
1706            self.clientPath = p4Where(self.depotPath)
1707
1708        if self.clientPath == "":
1709            die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1710
1711        print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1712        self.oldWorkingDirectory = os.getcwd()
1713
1714        # ensure the clientPath exists
1715        new_client_dir = False
1716        if not os.path.exists(self.clientPath):
1717            new_client_dir = True
1718            os.makedirs(self.clientPath)
1719
1720        chdir(self.clientPath, is_client_path=True)
1721        if self.dry_run:
1722            print "Would synchronize p4 checkout in %s" % self.clientPath
1723        else:
1724            print "Synchronizing p4 checkout..."
1725            if new_client_dir:
1726                # old one was destroyed, and maybe nobody told p4
1727                p4_sync("...", "-f")
1728            else:
1729                p4_sync("...")
1730        self.check()
1731
1732        commits = []
1733        for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1734            commits.append(line.strip())
1735        commits.reverse()
1736
1737        if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1738            self.checkAuthorship = False
1739        else:
1740            self.checkAuthorship = True
1741
1742        if self.preserveUser:
1743            self.checkValidP4Users(commits)
1744
1745        #
1746        # Build up a set of options to be passed to diff when
1747        # submitting each commit to p4.
1748        #
1749        if self.detectRenames:
1750            # command-line -M arg
1751            self.diffOpts = "-M"
1752        else:
1753            # If not explicitly set check the config variable
1754            detectRenames = gitConfig("git-p4.detectRenames")
1755
1756            if detectRenames.lower() == "false" or detectRenames == "":
1757                self.diffOpts = ""
1758            elif detectRenames.lower() == "true":
1759                self.diffOpts = "-M"
1760            else:
1761                self.diffOpts = "-M%s" % detectRenames
1762
1763        # no command-line arg for -C or --find-copies-harder, just
1764        # config variables
1765        detectCopies = gitConfig("git-p4.detectCopies")
1766        if detectCopies.lower() == "false" or detectCopies == "":
1767            pass
1768        elif detectCopies.lower() == "true":
1769            self.diffOpts += " -C"
1770        else:
1771            self.diffOpts += " -C%s" % detectCopies
1772
1773        if gitConfigBool("git-p4.detectCopiesHarder"):
1774            self.diffOpts += " --find-copies-harder"
1775
1776        #
1777        # Apply the commits, one at a time.  On failure, ask if should
1778        # continue to try the rest of the patches, or quit.
1779        #
1780        if self.dry_run:
1781            print "Would apply"
1782        applied = []
1783        last = len(commits) - 1
1784        for i, commit in enumerate(commits):
1785            if self.dry_run:
1786                print " ", read_pipe(["git", "show", "-s",
1787                                      "--format=format:%h %s", commit])
1788                ok = True
1789            else:
1790                ok = self.applyCommit(commit)
1791            if ok:
1792                applied.append(commit)
1793            else:
1794                if self.prepare_p4_only and i < last:
1795                    print "Processing only the first commit due to option" \
1796                          " --prepare-p4-only"
1797                    break
1798                if i < last:
1799                    quit = False
1800                    while True:
1801                        # prompt for what to do, or use the option/variable
1802                        if self.conflict_behavior == "ask":
1803                            print "What do you want to do?"
1804                            response = raw_input("[s]kip this commit but apply"
1805                                                 " the rest, or [q]uit? ")
1806                            if not response:
1807                                continue
1808                        elif self.conflict_behavior == "skip":
1809                            response = "s"
1810                        elif self.conflict_behavior == "quit":
1811                            response = "q"
1812                        else:
1813                            die("Unknown conflict_behavior '%s'" %
1814                                self.conflict_behavior)
1815
1816                        if response[0] == "s":
1817                            print "Skipping this commit, but applying the rest"
1818                            break
1819                        if response[0] == "q":
1820                            print "Quitting"
1821                            quit = True
1822                            break
1823                    if quit:
1824                        break
1825
1826        chdir(self.oldWorkingDirectory)
1827
1828        if self.dry_run:
1829            pass
1830        elif self.prepare_p4_only:
1831            pass
1832        elif len(commits) == len(applied):
1833            print "All commits applied!"
1834
1835            sync = P4Sync()
1836            if self.branch:
1837                sync.branch = self.branch
1838            sync.run([])
1839
1840            rebase = P4Rebase()
1841            rebase.rebase()
1842
1843        else:
1844            if len(applied) == 0:
1845                print "No commits applied."
1846            else:
1847                print "Applied only the commits marked with '*':"
1848                for c in commits:
1849                    if c in applied:
1850                        star = "*"
1851                    else:
1852                        star = " "
1853                    print star, read_pipe(["git", "show", "-s",
1854                                           "--format=format:%h %s",  c])
1855                print "You will have to do 'git p4 sync' and rebase."
1856
1857        if gitConfigBool("git-p4.exportLabels"):
1858            self.exportLabels = True
1859
1860        if self.exportLabels:
1861            p4Labels = getP4Labels(self.depotPath)
1862            gitTags = getGitTags()
1863
1864            missingGitTags = gitTags - p4Labels
1865            self.exportGitTags(missingGitTags)
1866
1867        # exit with error unless everything applied perfectly
1868        if len(commits) != len(applied):
1869                sys.exit(1)
1870
1871        return True
1872
1873class View(object):
1874    """Represent a p4 view ("p4 help views"), and map files in a
1875       repo according to the view."""
1876
1877    def __init__(self, client_name):
1878        self.mappings = []
1879        self.client_prefix = "//%s/" % client_name
1880        # cache results of "p4 where" to lookup client file locations
1881        self.client_spec_path_cache = {}
1882
1883    def append(self, view_line):
1884        """Parse a view line, splitting it into depot and client
1885           sides.  Append to self.mappings, preserving order.  This
1886           is only needed for tag creation."""
1887
1888        # Split the view line into exactly two words.  P4 enforces
1889        # structure on these lines that simplifies this quite a bit.
1890        #
1891        # Either or both words may be double-quoted.
1892        # Single quotes do not matter.
1893        # Double-quote marks cannot occur inside the words.
1894        # A + or - prefix is also inside the quotes.
1895        # There are no quotes unless they contain a space.
1896        # The line is already white-space stripped.
1897        # The two words are separated by a single space.
1898        #
1899        if view_line[0] == '"':
1900            # First word is double quoted.  Find its end.
1901            close_quote_index = view_line.find('"', 1)
1902            if close_quote_index <= 0:
1903                die("No first-word closing quote found: %s" % view_line)
1904            depot_side = view_line[1:close_quote_index]
1905            # skip closing quote and space
1906            rhs_index = close_quote_index + 1 + 1
1907        else:
1908            space_index = view_line.find(" ")
1909            if space_index <= 0:
1910                die("No word-splitting space found: %s" % view_line)
1911            depot_side = view_line[0:space_index]
1912            rhs_index = space_index + 1
1913
1914        # prefix + means overlay on previous mapping
1915        if depot_side.startswith("+"):
1916            depot_side = depot_side[1:]
1917
1918        # prefix - means exclude this path, leave out of mappings
1919        exclude = False
1920        if depot_side.startswith("-"):
1921            exclude = True
1922            depot_side = depot_side[1:]
1923
1924        if not exclude:
1925            self.mappings.append(depot_side)
1926
1927    def convert_client_path(self, clientFile):
1928        # chop off //client/ part to make it relative
1929        if not clientFile.startswith(self.client_prefix):
1930            die("No prefix '%s' on clientFile '%s'" %
1931                (self.client_prefix, clientFile))
1932        return clientFile[len(self.client_prefix):]
1933
1934    def update_client_spec_path_cache(self, files):
1935        """ Caching file paths by "p4 where" batch query """
1936
1937        # List depot file paths exclude that already cached
1938        fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1939
1940        if len(fileArgs) == 0:
1941            return  # All files in cache
1942
1943        where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1944        for res in where_result:
1945            if "code" in res and res["code"] == "error":
1946                # assume error is "... file(s) not in client view"
1947                continue
1948            if "clientFile" not in res:
1949                die("No clientFile in 'p4 where' output")
1950            if "unmap" in res:
1951                # it will list all of them, but only one not unmap-ped
1952                continue
1953            self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1954
1955        # not found files or unmap files set to ""
1956        for depotFile in fileArgs:
1957            if depotFile not in self.client_spec_path_cache:
1958                self.client_spec_path_cache[depotFile] = ""
1959
1960    def map_in_client(self, depot_path):
1961        """Return the relative location in the client where this
1962           depot file should live.  Returns "" if the file should
1963           not be mapped in the client."""
1964
1965        if depot_path in self.client_spec_path_cache:
1966            return self.client_spec_path_cache[depot_path]
1967
1968        die( "Error: %s is not found in client spec path" % depot_path )
1969        return ""
1970
1971class P4Sync(Command, P4UserMap):
1972    delete_actions = ( "delete", "move/delete", "purge" )
1973
1974    def __init__(self):
1975        Command.__init__(self)
1976        P4UserMap.__init__(self)
1977        self.options = [
1978                optparse.make_option("--branch", dest="branch"),
1979                optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1980                optparse.make_option("--changesfile", dest="changesFile"),
1981                optparse.make_option("--silent", dest="silent", action="store_true"),
1982                optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1983                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1984                optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1985                                     help="Import into refs/heads/ , not refs/remotes"),
1986                optparse.make_option("--max-changes", dest="maxChanges",
1987                                     help="Maximum number of changes to import"),
1988                optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
1989                                     help="Internal block size to use when iteratively calling p4 changes"),
1990                optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1991                                     help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1992                optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1993                                     help="Only sync files that are included in the Perforce Client Spec"),
1994                optparse.make_option("-/", dest="cloneExclude",
1995                                     action="append", type="string",
1996                                     help="exclude depot path"),
1997        ]
1998        self.description = """Imports from Perforce into a git repository.\n
1999    example:
2000    //depot/my/project/ -- to import the current head
2001    //depot/my/project/@all -- to import everything
2002    //depot/my/project/@1,6 -- to import only from revision 1 to 6
2003
2004    (a ... is not needed in the path p4 specification, it's added implicitly)"""
2005
2006        self.usage += " //depot/path[@revRange]"
2007        self.silent = False
2008        self.createdBranches = set()
2009        self.committedChanges = set()
2010        self.branch = ""
2011        self.detectBranches = False
2012        self.detectLabels = False
2013        self.importLabels = False
2014        self.changesFile = ""
2015        self.syncWithOrigin = True
2016        self.importIntoRemotes = True
2017        self.maxChanges = ""
2018        self.changes_block_size = None
2019        self.keepRepoPath = False
2020        self.depotPaths = None
2021        self.p4BranchesInGit = []
2022        self.cloneExclude = []
2023        self.useClientSpec = False
2024        self.useClientSpec_from_options = False
2025        self.clientSpecDirs = None
2026        self.tempBranches = []
2027        self.tempBranchLocation = "git-p4-tmp"
2028
2029        if gitConfig("git-p4.syncFromOrigin") == "false":
2030            self.syncWithOrigin = False
2031
2032    # This is required for the "append" cloneExclude action
2033    def ensure_value(self, attr, value):
2034        if not hasattr(self, attr) or getattr(self, attr) is None:
2035            setattr(self, attr, value)
2036        return getattr(self, attr)
2037
2038    # Force a checkpoint in fast-import and wait for it to finish
2039    def checkpoint(self):
2040        self.gitStream.write("checkpoint\n\n")
2041        self.gitStream.write("progress checkpoint\n\n")
2042        out = self.gitOutput.readline()
2043        if self.verbose:
2044            print "checkpoint finished: " + out
2045
2046    def extractFilesFromCommit(self, commit):
2047        self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2048                             for path in self.cloneExclude]
2049        files = []
2050        fnum = 0
2051        while commit.has_key("depotFile%s" % fnum):
2052            path =  commit["depotFile%s" % fnum]
2053
2054            if [p for p in self.cloneExclude
2055                if p4PathStartsWith(path, p)]:
2056                found = False
2057            else:
2058                found = [p for p in self.depotPaths
2059                         if p4PathStartsWith(path, p)]
2060            if not found:
2061                fnum = fnum + 1
2062                continue
2063
2064            file = {}
2065            file["path"] = path
2066            file["rev"] = commit["rev%s" % fnum]
2067            file["action"] = commit["action%s" % fnum]
2068            file["type"] = commit["type%s" % fnum]
2069            files.append(file)
2070            fnum = fnum + 1
2071        return files
2072
2073    def stripRepoPath(self, path, prefixes):
2074        """When streaming files, this is called to map a p4 depot path
2075           to where it should go in git.  The prefixes are either
2076           self.depotPaths, or self.branchPrefixes in the case of
2077           branch detection."""
2078
2079        if self.useClientSpec:
2080            # branch detection moves files up a level (the branch name)
2081            # from what client spec interpretation gives
2082            path = self.clientSpecDirs.map_in_client(path)
2083            if self.detectBranches:
2084                for b in self.knownBranches:
2085                    if path.startswith(b + "/"):
2086                        path = path[len(b)+1:]
2087
2088        elif self.keepRepoPath:
2089            # Preserve everything in relative path name except leading
2090            # //depot/; just look at first prefix as they all should
2091            # be in the same depot.
2092            depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2093            if p4PathStartsWith(path, depot):
2094                path = path[len(depot):]
2095
2096        else:
2097            for p in prefixes:
2098                if p4PathStartsWith(path, p):
2099                    path = path[len(p):]
2100                    break
2101
2102        path = wildcard_decode(path)
2103        return path
2104
2105    def splitFilesIntoBranches(self, commit):
2106        """Look at each depotFile in the commit to figure out to what
2107           branch it belongs."""
2108
2109        if self.clientSpecDirs:
2110            files = self.extractFilesFromCommit(commit)
2111            self.clientSpecDirs.update_client_spec_path_cache(files)
2112
2113        branches = {}
2114        fnum = 0
2115        while commit.has_key("depotFile%s" % fnum):
2116            path =  commit["depotFile%s" % fnum]
2117            found = [p for p in self.depotPaths
2118                     if p4PathStartsWith(path, p)]
2119            if not found:
2120                fnum = fnum + 1
2121                continue
2122
2123            file = {}
2124            file["path"] = path
2125            file["rev"] = commit["rev%s" % fnum]
2126            file["action"] = commit["action%s" % fnum]
2127            file["type"] = commit["type%s" % fnum]
2128            fnum = fnum + 1
2129
2130            # start with the full relative path where this file would
2131            # go in a p4 client
2132            if self.useClientSpec:
2133                relPath = self.clientSpecDirs.map_in_client(path)
2134            else:
2135                relPath = self.stripRepoPath(path, self.depotPaths)
2136
2137            for branch in self.knownBranches.keys():
2138                # add a trailing slash so that a commit into qt/4.2foo
2139                # doesn't end up in qt/4.2, e.g.
2140                if relPath.startswith(branch + "/"):
2141                    if branch not in branches:
2142                        branches[branch] = []
2143                    branches[branch].append(file)
2144                    break
2145
2146        return branches
2147
2148    # output one file from the P4 stream
2149    # - helper for streamP4Files
2150
2151    def streamOneP4File(self, file, contents):
2152        relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2153        if verbose:
2154            sys.stderr.write("%s\n" % relPath)
2155
2156        (type_base, type_mods) = split_p4_type(file["type"])
2157
2158        git_mode = "100644"
2159        if "x" in type_mods:
2160            git_mode = "100755"
2161        if type_base == "symlink":
2162            git_mode = "120000"
2163            # p4 print on a symlink sometimes contains "target\n";
2164            # if it does, remove the newline
2165            data = ''.join(contents)
2166            if not data:
2167                # Some version of p4 allowed creating a symlink that pointed
2168                # to nothing.  This causes p4 errors when checking out such
2169                # a change, and errors here too.  Work around it by ignoring
2170                # the bad symlink; hopefully a future change fixes it.
2171                print "\nIgnoring empty symlink in %s" % file['depotFile']
2172                return
2173            elif data[-1] == '\n':
2174                contents = [data[:-1]]
2175            else:
2176                contents = [data]
2177
2178        if type_base == "utf16":
2179            # p4 delivers different text in the python output to -G
2180            # than it does when using "print -o", or normal p4 client
2181            # operations.  utf16 is converted to ascii or utf8, perhaps.
2182            # But ascii text saved as -t utf16 is completely mangled.
2183            # Invoke print -o to get the real contents.
2184            #
2185            # On windows, the newlines will always be mangled by print, so put
2186            # them back too.  This is not needed to the cygwin windows version,
2187            # just the native "NT" type.
2188            #
2189            text = p4_read_pipe(['print', '-q', '-o', '-', "%s@%s" % (file['depotFile'], file['change']) ])
2190            if p4_version_string().find("/NT") >= 0:
2191                text = text.replace("\r\n", "\n")
2192            contents = [ text ]
2193
2194        if type_base == "apple":
2195            # Apple filetype files will be streamed as a concatenation of
2196            # its appledouble header and the contents.  This is useless
2197            # on both macs and non-macs.  If using "print -q -o xx", it
2198            # will create "xx" with the data, and "%xx" with the header.
2199            # This is also not very useful.
2200            #
2201            # Ideally, someday, this script can learn how to generate
2202            # appledouble files directly and import those to git, but
2203            # non-mac machines can never find a use for apple filetype.
2204            print "\nIgnoring apple filetype file %s" % file['depotFile']
2205            return
2206
2207        # Note that we do not try to de-mangle keywords on utf16 files,
2208        # even though in theory somebody may want that.
2209        pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2210        if pattern:
2211            regexp = re.compile(pattern, re.VERBOSE)
2212            text = ''.join(contents)
2213            text = regexp.sub(r'$\1$', text)
2214            contents = [ text ]
2215
2216        self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2217
2218        # total length...
2219        length = 0
2220        for d in contents:
2221            length = length + len(d)
2222
2223        self.gitStream.write("data %d\n" % length)
2224        for d in contents:
2225            self.gitStream.write(d)
2226        self.gitStream.write("\n")
2227
2228    def streamOneP4Deletion(self, file):
2229        relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2230        if verbose:
2231            sys.stderr.write("delete %s\n" % relPath)
2232        self.gitStream.write("D %s\n" % relPath)
2233
2234    # handle another chunk of streaming data
2235    def streamP4FilesCb(self, marshalled):
2236
2237        # catch p4 errors and complain
2238        err = None
2239        if "code" in marshalled:
2240            if marshalled["code"] == "error":
2241                if "data" in marshalled:
2242                    err = marshalled["data"].rstrip()
2243        if err:
2244            f = None
2245            if self.stream_have_file_info:
2246                if "depotFile" in self.stream_file:
2247                    f = self.stream_file["depotFile"]
2248            # force a failure in fast-import, else an empty
2249            # commit will be made
2250            self.gitStream.write("\n")
2251            self.gitStream.write("die-now\n")
2252            self.gitStream.close()
2253            # ignore errors, but make sure it exits first
2254            self.importProcess.wait()
2255            if f:
2256                die("Error from p4 print for %s: %s" % (f, err))
2257            else:
2258                die("Error from p4 print: %s" % err)
2259
2260        if marshalled.has_key('depotFile') and self.stream_have_file_info:
2261            # start of a new file - output the old one first
2262            self.streamOneP4File(self.stream_file, self.stream_contents)
2263            self.stream_file = {}
2264            self.stream_contents = []
2265            self.stream_have_file_info = False
2266
2267        # pick up the new file information... for the
2268        # 'data' field we need to append to our array
2269        for k in marshalled.keys():
2270            if k == 'data':
2271                self.stream_contents.append(marshalled['data'])
2272            else:
2273                self.stream_file[k] = marshalled[k]
2274
2275        self.stream_have_file_info = True
2276
2277    # Stream directly from "p4 files" into "git fast-import"
2278    def streamP4Files(self, files):
2279        filesForCommit = []
2280        filesToRead = []
2281        filesToDelete = []
2282
2283        for f in files:
2284            # if using a client spec, only add the files that have
2285            # a path in the client
2286            if self.clientSpecDirs:
2287                if self.clientSpecDirs.map_in_client(f['path']) == "":
2288                    continue
2289
2290            filesForCommit.append(f)
2291            if f['action'] in self.delete_actions:
2292                filesToDelete.append(f)
2293            else:
2294                filesToRead.append(f)
2295
2296        # deleted files...
2297        for f in filesToDelete:
2298            self.streamOneP4Deletion(f)
2299
2300        if len(filesToRead) > 0:
2301            self.stream_file = {}
2302            self.stream_contents = []
2303            self.stream_have_file_info = False
2304
2305            # curry self argument
2306            def streamP4FilesCbSelf(entry):
2307                self.streamP4FilesCb(entry)
2308
2309            fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2310
2311            p4CmdList(["-x", "-", "print"],
2312                      stdin=fileArgs,
2313                      cb=streamP4FilesCbSelf)
2314
2315            # do the last chunk
2316            if self.stream_file.has_key('depotFile'):
2317                self.streamOneP4File(self.stream_file, self.stream_contents)
2318
2319    def make_email(self, userid):
2320        if userid in self.users:
2321            return self.users[userid]
2322        else:
2323            return "%s <a@b>" % userid
2324
2325    # Stream a p4 tag
2326    def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2327        if verbose:
2328            print "writing tag %s for commit %s" % (labelName, commit)
2329        gitStream.write("tag %s\n" % labelName)
2330        gitStream.write("from %s\n" % commit)
2331
2332        if labelDetails.has_key('Owner'):
2333            owner = labelDetails["Owner"]
2334        else:
2335            owner = None
2336
2337        # Try to use the owner of the p4 label, or failing that,
2338        # the current p4 user id.
2339        if owner:
2340            email = self.make_email(owner)
2341        else:
2342            email = self.make_email(self.p4UserId())
2343        tagger = "%s %s %s" % (email, epoch, self.tz)
2344
2345        gitStream.write("tagger %s\n" % tagger)
2346
2347        print "labelDetails=",labelDetails
2348        if labelDetails.has_key('Description'):
2349            description = labelDetails['Description']
2350        else:
2351            description = 'Label from git p4'
2352
2353        gitStream.write("data %d\n" % len(description))
2354        gitStream.write(description)
2355        gitStream.write("\n")
2356
2357    def commit(self, details, files, branch, parent = ""):
2358        epoch = details["time"]
2359        author = details["user"]
2360
2361        if self.verbose:
2362            print "commit into %s" % branch
2363
2364        # start with reading files; if that fails, we should not
2365        # create a commit.
2366        new_files = []
2367        for f in files:
2368            if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2369                new_files.append (f)
2370            else:
2371                sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2372
2373        if self.clientSpecDirs:
2374            self.clientSpecDirs.update_client_spec_path_cache(files)
2375
2376        self.gitStream.write("commit %s\n" % branch)
2377#        gitStream.write("mark :%s\n" % details["change"])
2378        self.committedChanges.add(int(details["change"]))
2379        committer = ""
2380        if author not in self.users:
2381            self.getUserMapFromPerforceServer()
2382        committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2383
2384        self.gitStream.write("committer %s\n" % committer)
2385
2386        self.gitStream.write("data <<EOT\n")
2387        self.gitStream.write(details["desc"])
2388        self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2389                             (','.join(self.branchPrefixes), details["change"]))
2390        if len(details['options']) > 0:
2391            self.gitStream.write(": options = %s" % details['options'])
2392        self.gitStream.write("]\nEOT\n\n")
2393
2394        if len(parent) > 0:
2395            if self.verbose:
2396                print "parent %s" % parent
2397            self.gitStream.write("from %s\n" % parent)
2398
2399        self.streamP4Files(new_files)
2400        self.gitStream.write("\n")
2401
2402        change = int(details["change"])
2403
2404        if self.labels.has_key(change):
2405            label = self.labels[change]
2406            labelDetails = label[0]
2407            labelRevisions = label[1]
2408            if self.verbose:
2409                print "Change %s is labelled %s" % (change, labelDetails)
2410
2411            files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2412                                                for p in self.branchPrefixes])
2413
2414            if len(files) == len(labelRevisions):
2415
2416                cleanedFiles = {}
2417                for info in files:
2418                    if info["action"] in self.delete_actions:
2419                        continue
2420                    cleanedFiles[info["depotFile"]] = info["rev"]
2421
2422                if cleanedFiles == labelRevisions:
2423                    self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2424
2425                else:
2426                    if not self.silent:
2427                        print ("Tag %s does not match with change %s: files do not match."
2428                               % (labelDetails["label"], change))
2429
2430            else:
2431                if not self.silent:
2432                    print ("Tag %s does not match with change %s: file count is different."
2433                           % (labelDetails["label"], change))
2434
2435    # Build a dictionary of changelists and labels, for "detect-labels" option.
2436    def getLabels(self):
2437        self.labels = {}
2438
2439        l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2440        if len(l) > 0 and not self.silent:
2441            print "Finding files belonging to labels in %s" % `self.depotPaths`
2442
2443        for output in l:
2444            label = output["label"]
2445            revisions = {}
2446            newestChange = 0
2447            if self.verbose:
2448                print "Querying files for label %s" % label
2449            for file in p4CmdList(["files"] +
2450                                      ["%s...@%s" % (p, label)
2451                                          for p in self.depotPaths]):
2452                revisions[file["depotFile"]] = file["rev"]
2453                change = int(file["change"])
2454                if change > newestChange:
2455                    newestChange = change
2456
2457            self.labels[newestChange] = [output, revisions]
2458
2459        if self.verbose:
2460            print "Label changes: %s" % self.labels.keys()
2461
2462    # Import p4 labels as git tags. A direct mapping does not
2463    # exist, so assume that if all the files are at the same revision
2464    # then we can use that, or it's something more complicated we should
2465    # just ignore.
2466    def importP4Labels(self, stream, p4Labels):
2467        if verbose:
2468            print "import p4 labels: " + ' '.join(p4Labels)
2469
2470        ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2471        validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2472        if len(validLabelRegexp) == 0:
2473            validLabelRegexp = defaultLabelRegexp
2474        m = re.compile(validLabelRegexp)
2475
2476        for name in p4Labels:
2477            commitFound = False
2478
2479            if not m.match(name):
2480                if verbose:
2481                    print "label %s does not match regexp %s" % (name,validLabelRegexp)
2482                continue
2483
2484            if name in ignoredP4Labels:
2485                continue
2486
2487            labelDetails = p4CmdList(['label', "-o", name])[0]
2488
2489            # get the most recent changelist for each file in this label
2490            change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2491                                for p in self.depotPaths])
2492
2493            if change.has_key('change'):
2494                # find the corresponding git commit; take the oldest commit
2495                changelist = int(change['change'])
2496                gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2497                     "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2498                if len(gitCommit) == 0:
2499                    print "could not find git commit for changelist %d" % changelist
2500                else:
2501                    gitCommit = gitCommit.strip()
2502                    commitFound = True
2503                    # Convert from p4 time format
2504                    try:
2505                        tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2506                    except ValueError:
2507                        print "Could not convert label time %s" % labelDetails['Update']
2508                        tmwhen = 1
2509
2510                    when = int(time.mktime(tmwhen))
2511                    self.streamTag(stream, name, labelDetails, gitCommit, when)
2512                    if verbose:
2513                        print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2514            else:
2515                if verbose:
2516                    print "Label %s has no changelists - possibly deleted?" % name
2517
2518            if not commitFound:
2519                # We can't import this label; don't try again as it will get very
2520                # expensive repeatedly fetching all the files for labels that will
2521                # never be imported. If the label is moved in the future, the
2522                # ignore will need to be removed manually.
2523                system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2524
2525    def guessProjectName(self):
2526        for p in self.depotPaths:
2527            if p.endswith("/"):
2528                p = p[:-1]
2529            p = p[p.strip().rfind("/") + 1:]
2530            if not p.endswith("/"):
2531               p += "/"
2532            return p
2533
2534    def getBranchMapping(self):
2535        lostAndFoundBranches = set()
2536
2537        user = gitConfig("git-p4.branchUser")
2538        if len(user) > 0:
2539            command = "branches -u %s" % user
2540        else:
2541            command = "branches"
2542
2543        for info in p4CmdList(command):
2544            details = p4Cmd(["branch", "-o", info["branch"]])
2545            viewIdx = 0
2546            while details.has_key("View%s" % viewIdx):
2547                paths = details["View%s" % viewIdx].split(" ")
2548                viewIdx = viewIdx + 1
2549                # require standard //depot/foo/... //depot/bar/... mapping
2550                if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2551                    continue
2552                source = paths[0]
2553                destination = paths[1]
2554                ## HACK
2555                if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2556                    source = source[len(self.depotPaths[0]):-4]
2557                    destination = destination[len(self.depotPaths[0]):-4]
2558
2559                    if destination in self.knownBranches:
2560                        if not self.silent:
2561                            print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2562                            print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2563                        continue
2564
2565                    self.knownBranches[destination] = source
2566
2567                    lostAndFoundBranches.discard(destination)
2568
2569                    if source not in self.knownBranches:
2570                        lostAndFoundBranches.add(source)
2571
2572        # Perforce does not strictly require branches to be defined, so we also
2573        # check git config for a branch list.
2574        #
2575        # Example of branch definition in git config file:
2576        # [git-p4]
2577        #   branchList=main:branchA
2578        #   branchList=main:branchB
2579        #   branchList=branchA:branchC
2580        configBranches = gitConfigList("git-p4.branchList")
2581        for branch in configBranches:
2582            if branch:
2583                (source, destination) = branch.split(":")
2584                self.knownBranches[destination] = source
2585
2586                lostAndFoundBranches.discard(destination)
2587
2588                if source not in self.knownBranches:
2589                    lostAndFoundBranches.add(source)
2590
2591
2592        for branch in lostAndFoundBranches:
2593            self.knownBranches[branch] = branch
2594
2595    def getBranchMappingFromGitBranches(self):
2596        branches = p4BranchesInGit(self.importIntoRemotes)
2597        for branch in branches.keys():
2598            if branch == "master":
2599                branch = "main"
2600            else:
2601                branch = branch[len(self.projectName):]
2602            self.knownBranches[branch] = branch
2603
2604    def updateOptionDict(self, d):
2605        option_keys = {}
2606        if self.keepRepoPath:
2607            option_keys['keepRepoPath'] = 1
2608
2609        d["options"] = ' '.join(sorted(option_keys.keys()))
2610
2611    def readOptions(self, d):
2612        self.keepRepoPath = (d.has_key('options')
2613                             and ('keepRepoPath' in d['options']))
2614
2615    def gitRefForBranch(self, branch):
2616        if branch == "main":
2617            return self.refPrefix + "master"
2618
2619        if len(branch) <= 0:
2620            return branch
2621
2622        return self.refPrefix + self.projectName + branch
2623
2624    def gitCommitByP4Change(self, ref, change):
2625        if self.verbose:
2626            print "looking in ref " + ref + " for change %s using bisect..." % change
2627
2628        earliestCommit = ""
2629        latestCommit = parseRevision(ref)
2630
2631        while True:
2632            if self.verbose:
2633                print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2634            next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2635            if len(next) == 0:
2636                if self.verbose:
2637                    print "argh"
2638                return ""
2639            log = extractLogMessageFromGitCommit(next)
2640            settings = extractSettingsGitLog(log)
2641            currentChange = int(settings['change'])
2642            if self.verbose:
2643                print "current change %s" % currentChange
2644
2645            if currentChange == change:
2646                if self.verbose:
2647                    print "found %s" % next
2648                return next
2649
2650            if currentChange < change:
2651                earliestCommit = "^%s" % next
2652            else:
2653                latestCommit = "%s" % next
2654
2655        return ""
2656
2657    def importNewBranch(self, branch, maxChange):
2658        # make fast-import flush all changes to disk and update the refs using the checkpoint
2659        # command so that we can try to find the branch parent in the git history
2660        self.gitStream.write("checkpoint\n\n");
2661        self.gitStream.flush();
2662        branchPrefix = self.depotPaths[0] + branch + "/"
2663        range = "@1,%s" % maxChange
2664        #print "prefix" + branchPrefix
2665        changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
2666        if len(changes) <= 0:
2667            return False
2668        firstChange = changes[0]
2669        #print "first change in branch: %s" % firstChange
2670        sourceBranch = self.knownBranches[branch]
2671        sourceDepotPath = self.depotPaths[0] + sourceBranch
2672        sourceRef = self.gitRefForBranch(sourceBranch)
2673        #print "source " + sourceBranch
2674
2675        branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2676        #print "branch parent: %s" % branchParentChange
2677        gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2678        if len(gitParent) > 0:
2679            self.initialParents[self.gitRefForBranch(branch)] = gitParent
2680            #print "parent git commit: %s" % gitParent
2681
2682        self.importChanges(changes)
2683        return True
2684
2685    def searchParent(self, parent, branch, target):
2686        parentFound = False
2687        for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2688                                     "--no-merges", parent]):
2689            blob = blob.strip()
2690            if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2691                parentFound = True
2692                if self.verbose:
2693                    print "Found parent of %s in commit %s" % (branch, blob)
2694                break
2695        if parentFound:
2696            return blob
2697        else:
2698            return None
2699
2700    def importChanges(self, changes):
2701        cnt = 1
2702        for change in changes:
2703            description = p4_describe(change)
2704            self.updateOptionDict(description)
2705
2706            if not self.silent:
2707                sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2708                sys.stdout.flush()
2709            cnt = cnt + 1
2710
2711            try:
2712                if self.detectBranches:
2713                    branches = self.splitFilesIntoBranches(description)
2714                    for branch in branches.keys():
2715                        ## HACK  --hwn
2716                        branchPrefix = self.depotPaths[0] + branch + "/"
2717                        self.branchPrefixes = [ branchPrefix ]
2718
2719                        parent = ""
2720
2721                        filesForCommit = branches[branch]
2722
2723                        if self.verbose:
2724                            print "branch is %s" % branch
2725
2726                        self.updatedBranches.add(branch)
2727
2728                        if branch not in self.createdBranches:
2729                            self.createdBranches.add(branch)
2730                            parent = self.knownBranches[branch]
2731                            if parent == branch:
2732                                parent = ""
2733                            else:
2734                                fullBranch = self.projectName + branch
2735                                if fullBranch not in self.p4BranchesInGit:
2736                                    if not self.silent:
2737                                        print("\n    Importing new branch %s" % fullBranch);
2738                                    if self.importNewBranch(branch, change - 1):
2739                                        parent = ""
2740                                        self.p4BranchesInGit.append(fullBranch)
2741                                    if not self.silent:
2742                                        print("\n    Resuming with change %s" % change);
2743
2744                                if self.verbose:
2745                                    print "parent determined through known branches: %s" % parent
2746
2747                        branch = self.gitRefForBranch(branch)
2748                        parent = self.gitRefForBranch(parent)
2749
2750                        if self.verbose:
2751                            print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2752
2753                        if len(parent) == 0 and branch in self.initialParents:
2754                            parent = self.initialParents[branch]
2755                            del self.initialParents[branch]
2756
2757                        blob = None
2758                        if len(parent) > 0:
2759                            tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2760                            if self.verbose:
2761                                print "Creating temporary branch: " + tempBranch
2762                            self.commit(description, filesForCommit, tempBranch)
2763                            self.tempBranches.append(tempBranch)
2764                            self.checkpoint()
2765                            blob = self.searchParent(parent, branch, tempBranch)
2766                        if blob:
2767                            self.commit(description, filesForCommit, branch, blob)
2768                        else:
2769                            if self.verbose:
2770                                print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2771                            self.commit(description, filesForCommit, branch, parent)
2772                else:
2773                    files = self.extractFilesFromCommit(description)
2774                    self.commit(description, files, self.branch,
2775                                self.initialParent)
2776                    # only needed once, to connect to the previous commit
2777                    self.initialParent = ""
2778            except IOError:
2779                print self.gitError.read()
2780                sys.exit(1)
2781
2782    def importHeadRevision(self, revision):
2783        print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2784
2785        details = {}
2786        details["user"] = "git perforce import user"
2787        details["desc"] = ("Initial import of %s from the state at revision %s\n"
2788                           % (' '.join(self.depotPaths), revision))
2789        details["change"] = revision
2790        newestRevision = 0
2791
2792        fileCnt = 0
2793        fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2794
2795        for info in p4CmdList(["files"] + fileArgs):
2796
2797            if 'code' in info and info['code'] == 'error':
2798                sys.stderr.write("p4 returned an error: %s\n"
2799                                 % info['data'])
2800                if info['data'].find("must refer to client") >= 0:
2801                    sys.stderr.write("This particular p4 error is misleading.\n")
2802                    sys.stderr.write("Perhaps the depot path was misspelled.\n");
2803                    sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2804                sys.exit(1)
2805            if 'p4ExitCode' in info:
2806                sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2807                sys.exit(1)
2808
2809
2810            change = int(info["change"])
2811            if change > newestRevision:
2812                newestRevision = change
2813
2814            if info["action"] in self.delete_actions:
2815                # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2816                #fileCnt = fileCnt + 1
2817                continue
2818
2819            for prop in ["depotFile", "rev", "action", "type" ]:
2820                details["%s%s" % (prop, fileCnt)] = info[prop]
2821
2822            fileCnt = fileCnt + 1
2823
2824        details["change"] = newestRevision
2825
2826        # Use time from top-most change so that all git p4 clones of
2827        # the same p4 repo have the same commit SHA1s.
2828        res = p4_describe(newestRevision)
2829        details["time"] = res["time"]
2830
2831        self.updateOptionDict(details)
2832        try:
2833            self.commit(details, self.extractFilesFromCommit(details), self.branch)
2834        except IOError:
2835            print "IO error with git fast-import. Is your git version recent enough?"
2836            print self.gitError.read()
2837
2838
2839    def run(self, args):
2840        self.depotPaths = []
2841        self.changeRange = ""
2842        self.previousDepotPaths = []
2843        self.hasOrigin = False
2844
2845        # map from branch depot path to parent branch
2846        self.knownBranches = {}
2847        self.initialParents = {}
2848
2849        if self.importIntoRemotes:
2850            self.refPrefix = "refs/remotes/p4/"
2851        else:
2852            self.refPrefix = "refs/heads/p4/"
2853
2854        if self.syncWithOrigin:
2855            self.hasOrigin = originP4BranchesExist()
2856            if self.hasOrigin:
2857                if not self.silent:
2858                    print 'Syncing with origin first, using "git fetch origin"'
2859                system("git fetch origin")
2860
2861        branch_arg_given = bool(self.branch)
2862        if len(self.branch) == 0:
2863            self.branch = self.refPrefix + "master"
2864            if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2865                system("git update-ref %s refs/heads/p4" % self.branch)
2866                system("git branch -D p4")
2867
2868        # accept either the command-line option, or the configuration variable
2869        if self.useClientSpec:
2870            # will use this after clone to set the variable
2871            self.useClientSpec_from_options = True
2872        else:
2873            if gitConfigBool("git-p4.useclientspec"):
2874                self.useClientSpec = True
2875        if self.useClientSpec:
2876            self.clientSpecDirs = getClientSpec()
2877
2878        # TODO: should always look at previous commits,
2879        # merge with previous imports, if possible.
2880        if args == []:
2881            if self.hasOrigin:
2882                createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2883
2884            # branches holds mapping from branch name to sha1
2885            branches = p4BranchesInGit(self.importIntoRemotes)
2886
2887            # restrict to just this one, disabling detect-branches
2888            if branch_arg_given:
2889                short = self.branch.split("/")[-1]
2890                if short in branches:
2891                    self.p4BranchesInGit = [ short ]
2892            else:
2893                self.p4BranchesInGit = branches.keys()
2894
2895            if len(self.p4BranchesInGit) > 1:
2896                if not self.silent:
2897                    print "Importing from/into multiple branches"
2898                self.detectBranches = True
2899                for branch in branches.keys():
2900                    self.initialParents[self.refPrefix + branch] = \
2901                        branches[branch]
2902
2903            if self.verbose:
2904                print "branches: %s" % self.p4BranchesInGit
2905
2906            p4Change = 0
2907            for branch in self.p4BranchesInGit:
2908                logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2909
2910                settings = extractSettingsGitLog(logMsg)
2911
2912                self.readOptions(settings)
2913                if (settings.has_key('depot-paths')
2914                    and settings.has_key ('change')):
2915                    change = int(settings['change']) + 1
2916                    p4Change = max(p4Change, change)
2917
2918                    depotPaths = sorted(settings['depot-paths'])
2919                    if self.previousDepotPaths == []:
2920                        self.previousDepotPaths = depotPaths
2921                    else:
2922                        paths = []
2923                        for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2924                            prev_list = prev.split("/")
2925                            cur_list = cur.split("/")
2926                            for i in range(0, min(len(cur_list), len(prev_list))):
2927                                if cur_list[i] <> prev_list[i]:
2928                                    i = i - 1
2929                                    break
2930
2931                            paths.append ("/".join(cur_list[:i + 1]))
2932
2933                        self.previousDepotPaths = paths
2934
2935            if p4Change > 0:
2936                self.depotPaths = sorted(self.previousDepotPaths)
2937                self.changeRange = "@%s,#head" % p4Change
2938                if not self.silent and not self.detectBranches:
2939                    print "Performing incremental import into %s git branch" % self.branch
2940
2941        # accept multiple ref name abbreviations:
2942        #    refs/foo/bar/branch -> use it exactly
2943        #    p4/branch -> prepend refs/remotes/ or refs/heads/
2944        #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2945        if not self.branch.startswith("refs/"):
2946            if self.importIntoRemotes:
2947                prepend = "refs/remotes/"
2948            else:
2949                prepend = "refs/heads/"
2950            if not self.branch.startswith("p4/"):
2951                prepend += "p4/"
2952            self.branch = prepend + self.branch
2953
2954        if len(args) == 0 and self.depotPaths:
2955            if not self.silent:
2956                print "Depot paths: %s" % ' '.join(self.depotPaths)
2957        else:
2958            if self.depotPaths and self.depotPaths != args:
2959                print ("previous import used depot path %s and now %s was specified. "
2960                       "This doesn't work!" % (' '.join (self.depotPaths),
2961                                               ' '.join (args)))
2962                sys.exit(1)
2963
2964            self.depotPaths = sorted(args)
2965
2966        revision = ""
2967        self.users = {}
2968
2969        # Make sure no revision specifiers are used when --changesfile
2970        # is specified.
2971        bad_changesfile = False
2972        if len(self.changesFile) > 0:
2973            for p in self.depotPaths:
2974                if p.find("@") >= 0 or p.find("#") >= 0:
2975                    bad_changesfile = True
2976                    break
2977        if bad_changesfile:
2978            die("Option --changesfile is incompatible with revision specifiers")
2979
2980        newPaths = []
2981        for p in self.depotPaths:
2982            if p.find("@") != -1:
2983                atIdx = p.index("@")
2984                self.changeRange = p[atIdx:]
2985                if self.changeRange == "@all":
2986                    self.changeRange = ""
2987                elif ',' not in self.changeRange:
2988                    revision = self.changeRange
2989                    self.changeRange = ""
2990                p = p[:atIdx]
2991            elif p.find("#") != -1:
2992                hashIdx = p.index("#")
2993                revision = p[hashIdx:]
2994                p = p[:hashIdx]
2995            elif self.previousDepotPaths == []:
2996                # pay attention to changesfile, if given, else import
2997                # the entire p4 tree at the head revision
2998                if len(self.changesFile) == 0:
2999                    revision = "#head"
3000
3001            p = re.sub ("\.\.\.$", "", p)
3002            if not p.endswith("/"):
3003                p += "/"
3004
3005            newPaths.append(p)
3006
3007        self.depotPaths = newPaths
3008
3009        # --detect-branches may change this for each branch
3010        self.branchPrefixes = self.depotPaths
3011
3012        self.loadUserMapFromCache()
3013        self.labels = {}
3014        if self.detectLabels:
3015            self.getLabels();
3016
3017        if self.detectBranches:
3018            ## FIXME - what's a P4 projectName ?
3019            self.projectName = self.guessProjectName()
3020
3021            if self.hasOrigin:
3022                self.getBranchMappingFromGitBranches()
3023            else:
3024                self.getBranchMapping()
3025            if self.verbose:
3026                print "p4-git branches: %s" % self.p4BranchesInGit
3027                print "initial parents: %s" % self.initialParents
3028            for b in self.p4BranchesInGit:
3029                if b != "master":
3030
3031                    ## FIXME
3032                    b = b[len(self.projectName):]
3033                self.createdBranches.add(b)
3034
3035        self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3036
3037        self.importProcess = subprocess.Popen(["git", "fast-import"],
3038                                              stdin=subprocess.PIPE,
3039                                              stdout=subprocess.PIPE,
3040                                              stderr=subprocess.PIPE);
3041        self.gitOutput = self.importProcess.stdout
3042        self.gitStream = self.importProcess.stdin
3043        self.gitError = self.importProcess.stderr
3044
3045        if revision:
3046            self.importHeadRevision(revision)
3047        else:
3048            changes = []
3049
3050            if len(self.changesFile) > 0:
3051                output = open(self.changesFile).readlines()
3052                changeSet = set()
3053                for line in output:
3054                    changeSet.add(int(line))
3055
3056                for change in changeSet:
3057                    changes.append(change)
3058
3059                changes.sort()
3060            else:
3061                # catch "git p4 sync" with no new branches, in a repo that
3062                # does not have any existing p4 branches
3063                if len(args) == 0:
3064                    if not self.p4BranchesInGit:
3065                        die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3066
3067                    # The default branch is master, unless --branch is used to
3068                    # specify something else.  Make sure it exists, or complain
3069                    # nicely about how to use --branch.
3070                    if not self.detectBranches:
3071                        if not branch_exists(self.branch):
3072                            if branch_arg_given:
3073                                die("Error: branch %s does not exist." % self.branch)
3074                            else:
3075                                die("Error: no branch %s; perhaps specify one with --branch." %
3076                                    self.branch)
3077
3078                if self.verbose:
3079                    print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3080                                                              self.changeRange)
3081                changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3082
3083                if len(self.maxChanges) > 0:
3084                    changes = changes[:min(int(self.maxChanges), len(changes))]
3085
3086            if len(changes) == 0:
3087                if not self.silent:
3088                    print "No changes to import!"
3089            else:
3090                if not self.silent and not self.detectBranches:
3091                    print "Import destination: %s" % self.branch
3092
3093                self.updatedBranches = set()
3094
3095                if not self.detectBranches:
3096                    if args:
3097                        # start a new branch
3098                        self.initialParent = ""
3099                    else:
3100                        # build on a previous revision
3101                        self.initialParent = parseRevision(self.branch)
3102
3103                self.importChanges(changes)
3104
3105                if not self.silent:
3106                    print ""
3107                    if len(self.updatedBranches) > 0:
3108                        sys.stdout.write("Updated branches: ")
3109                        for b in self.updatedBranches:
3110                            sys.stdout.write("%s " % b)
3111                        sys.stdout.write("\n")
3112
3113        if gitConfigBool("git-p4.importLabels"):
3114            self.importLabels = True
3115
3116        if self.importLabels:
3117            p4Labels = getP4Labels(self.depotPaths)
3118            gitTags = getGitTags()
3119
3120            missingP4Labels = p4Labels - gitTags
3121            self.importP4Labels(self.gitStream, missingP4Labels)
3122
3123        self.gitStream.close()
3124        if self.importProcess.wait() != 0:
3125            die("fast-import failed: %s" % self.gitError.read())
3126        self.gitOutput.close()
3127        self.gitError.close()
3128
3129        # Cleanup temporary branches created during import
3130        if self.tempBranches != []:
3131            for branch in self.tempBranches:
3132                read_pipe("git update-ref -d %s" % branch)
3133            os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3134
3135        # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3136        # a convenient shortcut refname "p4".
3137        if self.importIntoRemotes:
3138            head_ref = self.refPrefix + "HEAD"
3139            if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3140                system(["git", "symbolic-ref", head_ref, self.branch])
3141
3142        return True
3143
3144class P4Rebase(Command):
3145    def __init__(self):
3146        Command.__init__(self)
3147        self.options = [
3148                optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3149        ]
3150        self.importLabels = False
3151        self.description = ("Fetches the latest revision from perforce and "
3152                            + "rebases the current work (branch) against it")
3153
3154    def run(self, args):
3155        sync = P4Sync()
3156        sync.importLabels = self.importLabels
3157        sync.run([])
3158
3159        return self.rebase()
3160
3161    def rebase(self):
3162        if os.system("git update-index --refresh") != 0:
3163            die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3164        if len(read_pipe("git diff-index HEAD --")) > 0:
3165            die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3166
3167        [upstream, settings] = findUpstreamBranchPoint()
3168        if len(upstream) == 0:
3169            die("Cannot find upstream branchpoint for rebase")
3170
3171        # the branchpoint may be p4/foo~3, so strip off the parent
3172        upstream = re.sub("~[0-9]+$", "", upstream)
3173
3174        print "Rebasing the current branch onto %s" % upstream
3175        oldHead = read_pipe("git rev-parse HEAD").strip()
3176        system("git rebase %s" % upstream)
3177        system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3178        return True
3179
3180class P4Clone(P4Sync):
3181    def __init__(self):
3182        P4Sync.__init__(self)
3183        self.description = "Creates a new git repository and imports from Perforce into it"
3184        self.usage = "usage: %prog [options] //depot/path[@revRange]"
3185        self.options += [
3186            optparse.make_option("--destination", dest="cloneDestination",
3187                                 action='store', default=None,
3188                                 help="where to leave result of the clone"),
3189            optparse.make_option("--bare", dest="cloneBare",
3190                                 action="store_true", default=False),
3191        ]
3192        self.cloneDestination = None
3193        self.needsGit = False
3194        self.cloneBare = False
3195
3196    def defaultDestination(self, args):
3197        ## TODO: use common prefix of args?
3198        depotPath = args[0]
3199        depotDir = re.sub("(@[^@]*)$", "", depotPath)
3200        depotDir = re.sub("(#[^#]*)$", "", depotDir)
3201        depotDir = re.sub(r"\.\.\.$", "", depotDir)
3202        depotDir = re.sub(r"/$", "", depotDir)
3203        return os.path.split(depotDir)[1]
3204
3205    def run(self, args):
3206        if len(args) < 1:
3207            return False
3208
3209        if self.keepRepoPath and not self.cloneDestination:
3210            sys.stderr.write("Must specify destination for --keep-path\n")
3211            sys.exit(1)
3212
3213        depotPaths = args
3214
3215        if not self.cloneDestination and len(depotPaths) > 1:
3216            self.cloneDestination = depotPaths[-1]
3217            depotPaths = depotPaths[:-1]
3218
3219        self.cloneExclude = ["/"+p for p in self.cloneExclude]
3220        for p in depotPaths:
3221            if not p.startswith("//"):
3222                sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3223                return False
3224
3225        if not self.cloneDestination:
3226            self.cloneDestination = self.defaultDestination(args)
3227
3228        print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3229
3230        if not os.path.exists(self.cloneDestination):
3231            os.makedirs(self.cloneDestination)
3232        chdir(self.cloneDestination)
3233
3234        init_cmd = [ "git", "init" ]
3235        if self.cloneBare:
3236            init_cmd.append("--bare")
3237        retcode = subprocess.call(init_cmd)
3238        if retcode:
3239            raise CalledProcessError(retcode, init_cmd)
3240
3241        if not P4Sync.run(self, depotPaths):
3242            return False
3243
3244        # create a master branch and check out a work tree
3245        if gitBranchExists(self.branch):
3246            system([ "git", "branch", "master", self.branch ])
3247            if not self.cloneBare:
3248                system([ "git", "checkout", "-f" ])
3249        else:
3250            print 'Not checking out any branch, use ' \
3251                  '"git checkout -q -b master <branch>"'
3252
3253        # auto-set this variable if invoked with --use-client-spec
3254        if self.useClientSpec_from_options:
3255            system("git config --bool git-p4.useclientspec true")
3256
3257        return True
3258
3259class P4Branches(Command):
3260    def __init__(self):
3261        Command.__init__(self)
3262        self.options = [ ]
3263        self.description = ("Shows the git branches that hold imports and their "
3264                            + "corresponding perforce depot paths")
3265        self.verbose = False
3266
3267    def run(self, args):
3268        if originP4BranchesExist():
3269            createOrUpdateBranchesFromOrigin()
3270
3271        cmdline = "git rev-parse --symbolic "
3272        cmdline += " --remotes"
3273
3274        for line in read_pipe_lines(cmdline):
3275            line = line.strip()
3276
3277            if not line.startswith('p4/') or line == "p4/HEAD":
3278                continue
3279            branch = line
3280
3281            log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3282            settings = extractSettingsGitLog(log)
3283
3284            print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3285        return True
3286
3287class HelpFormatter(optparse.IndentedHelpFormatter):
3288    def __init__(self):
3289        optparse.IndentedHelpFormatter.__init__(self)
3290
3291    def format_description(self, description):
3292        if description:
3293            return description + "\n"
3294        else:
3295            return ""
3296
3297def printUsage(commands):
3298    print "usage: %s <command> [options]" % sys.argv[0]
3299    print ""
3300    print "valid commands: %s" % ", ".join(commands)
3301    print ""
3302    print "Try %s <command> --help for command specific help." % sys.argv[0]
3303    print ""
3304
3305commands = {
3306    "debug" : P4Debug,
3307    "submit" : P4Submit,
3308    "commit" : P4Submit,
3309    "sync" : P4Sync,
3310    "rebase" : P4Rebase,
3311    "clone" : P4Clone,
3312    "rollback" : P4RollBack,
3313    "branches" : P4Branches
3314}
3315
3316
3317def main():
3318    if len(sys.argv[1:]) == 0:
3319        printUsage(commands.keys())
3320        sys.exit(2)
3321
3322    cmdName = sys.argv[1]
3323    try:
3324        klass = commands[cmdName]
3325        cmd = klass()
3326    except KeyError:
3327        print "unknown command %s" % cmdName
3328        print ""
3329        printUsage(commands.keys())
3330        sys.exit(2)
3331
3332    options = cmd.options
3333    cmd.gitdir = os.environ.get("GIT_DIR", None)
3334
3335    args = sys.argv[2:]
3336
3337    options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3338    if cmd.needsGit:
3339        options.append(optparse.make_option("--git-dir", dest="gitdir"))
3340
3341    parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3342                                   options,
3343                                   description = cmd.description,
3344                                   formatter = HelpFormatter())
3345
3346    (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3347    global verbose
3348    verbose = cmd.verbose
3349    if cmd.needsGit:
3350        if cmd.gitdir == None:
3351            cmd.gitdir = os.path.abspath(".git")
3352            if not isValidGitDir(cmd.gitdir):
3353                cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3354                if os.path.exists(cmd.gitdir):
3355                    cdup = read_pipe("git rev-parse --show-cdup").strip()
3356                    if len(cdup) > 0:
3357                        chdir(cdup);
3358
3359        if not isValidGitDir(cmd.gitdir):
3360            if isValidGitDir(cmd.gitdir + "/.git"):
3361                cmd.gitdir += "/.git"
3362            else:
3363                die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3364
3365        os.environ["GIT_DIR"] = cmd.gitdir
3366
3367    if not cmd.run(args):
3368        parser.print_help()
3369        sys.exit(2)
3370
3371
3372if __name__ == '__main__':
3373    main()